PetscErrorCode VecLoad_Binary(Vec vec, PetscViewer viewer)
{
PetscMPIInt size,rank,tag;
int fd;
PetscInt i,rows = 0,n,*range,N,bs;
PetscErrorCode ierr;
PetscBool flag;
PetscScalar *avec,*avecwork;
MPI_Comm comm;
MPI_Request request;
MPI_Status status;
#if defined(PETSC_HAVE_MPIIO)
PetscBool useMPIIO;
#endif
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(viewer,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryReadVecHeader_Private(viewer,&rows);CHKERRQ(ierr);
/* Set Vec sizes,blocksize,and type if not already set. Block size first so that local sizes will be compatible. */
ierr = PetscOptionsGetInt(((PetscObject)vec)->prefix, "-vecload_block_size", &bs, &flag);CHKERRQ(ierr);
if (flag) {
ierr = VecSetBlockSize(vec, bs);CHKERRQ(ierr);
}
if (vec->map->n < 0 && vec->map->N < 0) {
ierr = VecSetSizes(vec,PETSC_DECIDE,rows);CHKERRQ(ierr);
}
/* If sizes and type already set,check if the vector global size is correct */
ierr = VecGetSize(vec, &N);CHKERRQ(ierr);
if (N != rows) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Vector in file different length (%d) then input vector (%d)", rows, N);
#if defined(PETSC_HAVE_MPIIO)
ierr = PetscViewerBinaryGetMPIIO(viewer,&useMPIIO);CHKERRQ(ierr);
if (useMPIIO) {
ierr = VecLoad_Binary_MPIIO(vec, viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
ierr = VecGetLocalSize(vec,&n);CHKERRQ(ierr);
ierr = PetscObjectGetNewTag((PetscObject)viewer,&tag);CHKERRQ(ierr);
ierr = VecGetArray(vec,&avec);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryRead(fd,avec,n,PETSC_SCALAR);CHKERRQ(ierr);
if (size > 1) {
/* read in other chuncks and send to other processors */
/* determine maximum chunck owned by other */
range = vec->map->range;
n = 1;
for (i=1; i<size; i++) n = PetscMax(n,range[i+1] - range[i]);
ierr = PetscMalloc1(n,&avecwork);CHKERRQ(ierr);
for (i=1; i<size; i++) {
n = range[i+1] - range[i];
ierr = PetscBinaryRead(fd,avecwork,n,PETSC_SCALAR);CHKERRQ(ierr);
ierr = MPI_Isend(avecwork,n,MPIU_SCALAR,i,tag,comm,&request);CHKERRQ(ierr);
ierr = MPI_Wait(&request,&status);CHKERRQ(ierr);
}
ierr = PetscFree(avecwork);CHKERRQ(ierr);
}
} else {
ierr = MPI_Recv(avec,n,MPIU_SCALAR,0,tag,comm,&status);CHKERRQ(ierr);
}
ierr = VecRestoreArray(vec,&avec);CHKERRQ(ierr);
ierr = VecAssemblyBegin(vec);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode MatGetSubMatrices_MPIAdj_Private(Mat mat,PetscInt n,const IS irow[],const IS icol[],PetscBool subcomm,MatReuse scall,Mat *submat[])
{
PetscInt i,irow_n,icol_n,*sxadj,*sadjncy,*svalues;
PetscInt *indices,nindx,j,k,loc;
PetscMPIInt issame;
const PetscInt *irow_indices,*icol_indices;
MPI_Comm scomm_row,scomm_col,scomm_mat;
PetscErrorCode ierr;
PetscFunctionBegin;
nindx = 0;
/*
* Estimate a maximum number for allocating memory
*/
for(i=0; i<n; i++){
ierr = ISGetLocalSize(irow[i],&irow_n);CHKERRQ(ierr);
ierr = ISGetLocalSize(icol[i],&icol_n);CHKERRQ(ierr);
nindx = nindx>(irow_n+icol_n)? nindx:(irow_n+icol_n);
}
ierr = PetscCalloc1(nindx,&indices);CHKERRQ(ierr);
/* construct a submat */
for(i=0; i<n; i++){
/*comms */
if(subcomm){
ierr = PetscObjectGetComm((PetscObject)irow[i],&scomm_row);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)icol[i],&scomm_col);CHKERRQ(ierr);
ierr = MPI_Comm_compare(scomm_row,scomm_col,&issame);CHKERRQ(ierr);
if(issame != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"row index set must have the same comm as the col index set\n");
ierr = MPI_Comm_compare(scomm_row,PETSC_COMM_SELF,&issame);CHKERRQ(ierr);
if(issame == MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP," can not use PETSC_COMM_SELF as comm when extracting a parallel submatrix\n");
}else{
scomm_row = PETSC_COMM_SELF;
}
/*get sub-matrix data*/
sxadj=0; sadjncy=0; svalues=0;
ierr = MatGetSubMatrix_MPIAdj_data(mat,irow[i],icol[i],&sxadj,&sadjncy,&svalues);CHKERRQ(ierr);
ierr = ISGetLocalSize(irow[i],&irow_n);CHKERRQ(ierr);
ierr = ISGetLocalSize(icol[i],&icol_n);CHKERRQ(ierr);
ierr = ISGetIndices(irow[i],&irow_indices);CHKERRQ(ierr);
ierr = PetscMemcpy(indices,irow_indices,sizeof(PetscInt)*irow_n);CHKERRQ(ierr);
ierr = ISRestoreIndices(irow[i],&irow_indices);CHKERRQ(ierr);
ierr = ISGetIndices(icol[i],&icol_indices);CHKERRQ(ierr);
ierr = PetscMemcpy(indices+irow_n,icol_indices,sizeof(PetscInt)*icol_n);CHKERRQ(ierr);
ierr = ISRestoreIndices(icol[i],&icol_indices);CHKERRQ(ierr);
nindx = irow_n+icol_n;
ierr = PetscSortRemoveDupsInt(&nindx,indices);CHKERRQ(ierr);
/* renumber columns */
for(j=0; j<irow_n; j++){
for(k=sxadj[j]; k<sxadj[j+1]; k++){
ierr = PetscFindInt(sadjncy[k],nindx,indices,&loc);CHKERRQ(ierr);
#if PETSC_USE_DEBUG
if(loc<0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"can not find col %d \n",sadjncy[k]);
#endif
sadjncy[k] = loc;
}
}
if(scall==MAT_INITIAL_MATRIX){
ierr = MatCreateMPIAdj(scomm_row,irow_n,icol_n,sxadj,sadjncy,svalues,submat[i]);CHKERRQ(ierr);
}else{
Mat sadj = *(submat[i]);
Mat_MPIAdj *sa = (Mat_MPIAdj*)((sadj)->data);
ierr = PetscObjectGetComm((PetscObject)sadj,&scomm_mat);CHKERRQ(ierr);
ierr = MPI_Comm_compare(scomm_row,scomm_mat,&issame);CHKERRQ(ierr);
if(issame != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"submatrix must have the same comm as the col index set\n");
ierr = PetscMemcpy(sa->i,sxadj,sizeof(PetscInt)*(irow_n+1));CHKERRQ(ierr);
ierr = PetscMemcpy(sa->j,sadjncy,sizeof(PetscInt)*sxadj[irow_n]);CHKERRQ(ierr);
if(svalues){ierr = PetscMemcpy(sa->values,svalues,sizeof(PetscInt)*sxadj[irow_n]);CHKERRQ(ierr);}
ierr = PetscFree(sxadj);CHKERRQ(ierr);
ierr = PetscFree(sadjncy);CHKERRQ(ierr);
if(svalues) {ierr = PetscFree(svalues);CHKERRQ(ierr);}
}
}
ierr = PetscFree(indices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,B;
Vec xx,s1,s2,yy;
PetscErrorCode ierr;
PetscInt m=45,rows[2],cols[2],bs=1,i,row,col,*idx,M;
PetscScalar rval,vals1[4],vals2[4];
PetscRandom rdm;
IS is1,is2;
PetscReal s1norm,s2norm,rnorm,tol = 1.e-4;
PetscBool flg;
MatFactorInfo info;
PetscInitialize(&argc,&args,(char *)0,help);
/* Test MatSetValues() and MatGetValues() */
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_block_size",&bs,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_size",&m,PETSC_NULL);CHKERRQ(ierr);
M = m*bs;
ierr = MatCreateSeqBAIJ(PETSC_COMM_SELF,bs,M,M,1,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,M,M,15,PETSC_NULL,&B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,M,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&yy);CHKERRQ(ierr);
/* For each row add atleast 15 elements */
for (row=0; row<M; row++) {
for (i=0; i<25*bs; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
col = (PetscInt)(PetscRealPart(rval)*M);
ierr = MatSetValues(A,1,&row,1,&col,&rval,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(B,1,&row,1,&col,&rval,INSERT_VALUES);CHKERRQ(ierr);
}
}
/* Now set blocks of values */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[0] = (PetscInt)(PetscRealPart(rval)*M);
vals1[0] = rval;
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[1] = (PetscInt)(PetscRealPart(rval)*M);
vals1[1] = rval;
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[0] = (PetscInt)(PetscRealPart(rval)*M);
vals1[2] = rval;
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[1] = (PetscInt)(PetscRealPart(rval)*M);
vals1[3] = rval;
ierr = MatSetValues(A,2,rows,2,cols,vals1,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(B,2,rows,2,cols,vals1,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&s1norm);CHKERRQ(ierr);
ierr = MatNorm(B,NORM_FROBENIUS,&s2norm);CHKERRQ(ierr);
rnorm = PetscAbsScalar(s2norm-s1norm)/s2norm;
if ( rnorm>tol ) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS()- NormA=%16.14e NormB=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&s1norm);CHKERRQ(ierr);
ierr = MatNorm(B,NORM_INFINITY,&s2norm);CHKERRQ(ierr);
rnorm = PetscAbsScalar(s2norm-s1norm)/s2norm;
if ( rnorm>tol ) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY()- NormA=%16.14e NormB=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&s1norm);CHKERRQ(ierr);
ierr = MatNorm(B,NORM_1,&s2norm);CHKERRQ(ierr);
rnorm = PetscAbsScalar(s2norm-s1norm)/s2norm;
if ( rnorm>tol ) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_NORM_1()- NormA=%16.14e NormB=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
/* MatShift() */
rval = 10*s1norm;
ierr = MatShift(A,rval);CHKERRQ(ierr);
ierr = MatShift(B,rval);CHKERRQ(ierr);
/* Test MatTranspose() */
ierr = MatTranspose(A,MAT_REUSE_MATRIX,&A);CHKERRQ(ierr);
ierr = MatTranspose(B,MAT_REUSE_MATRIX,&B);CHKERRQ(ierr);
/* Now do MatGetValues() */
for (i=0; i<30; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[0] = (PetscInt)(PetscRealPart(rval)*M);
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[1] = (PetscInt)(PetscRealPart(rval)*M);
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[0] = (PetscInt)(PetscRealPart(rval)*M);
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[1] = (PetscInt)(PetscRealPart(rval)*M);
ierr = MatGetValues(A,2,rows,2,cols,vals1);CHKERRQ(ierr);
ierr = MatGetValues(B,2,rows,2,cols,vals2);CHKERRQ(ierr);
ierr = PetscMemcmp(vals1,vals2,4*sizeof(PetscScalar),&flg);CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetValues bs = %D\n",bs);CHKERRQ(ierr);
}
}
/* Test MatMult(), MatMultAdd() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = VecSet(s2,0.0);CHKERRQ(ierr);
ierr = MatMult(A,xx,s1);CHKERRQ(ierr);
ierr = MatMultAdd(A,xx,s2,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatMult not equal to MatMultAdd Norm1=%e Norm2=%e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
}
/* Test MatMult() */
ierr = MatMultEqual(A,B,10,&flg);CHKERRQ(ierr);
if (!flg){
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMult()\n");CHKERRQ(ierr);
}
/* Test MatMultAdd() */
ierr = MatMultAddEqual(A,B,10,&flg);CHKERRQ(ierr);
if (!flg){
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMultAdd()\n");CHKERRQ(ierr);
}
/* Test MatMultTranspose() */
ierr = MatMultTransposeEqual(A,B,10,&flg);CHKERRQ(ierr);
if (!flg){
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMultTranspose()\n");CHKERRQ(ierr);
}
/* Test MatMultTransposeAdd() */
ierr = MatMultTransposeAddEqual(A,B,10,&flg);CHKERRQ(ierr);
if (!flg){
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMultTransposeAdd()\n");CHKERRQ(ierr);
}
/* Do LUFactor() on both the matrices */
ierr = PetscMalloc(M*sizeof(PetscInt),&idx);CHKERRQ(ierr);
for (i=0; i<M; i++) idx[i] = i;
ierr = ISCreateGeneral(PETSC_COMM_SELF,M,idx,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,M,idx,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = ISSetPermutation(is1);CHKERRQ(ierr);
ierr = ISSetPermutation(is2);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);CHKERRQ(ierr);
info.fill = 2.0;
info.dtcol = 0.0;
info.zeropivot = 1.e-14;
info.pivotinblocks = 1.0;
ierr = MatLUFactor(B,is1,is2,&info);CHKERRQ(ierr);
ierr = MatLUFactor(A,is1,is2,&info);CHKERRQ(ierr);
/* Test MatSolveAdd() */
for (i=0; i<10; i++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = VecSetRandom(yy,rdm);CHKERRQ(ierr);
ierr = MatSolveAdd(B,xx,yy,s2);CHKERRQ(ierr);
ierr = MatSolveAdd(A,xx,yy,s1);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatSolveAdd - Norm1=%16.14e Norm2=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
}
/* Test MatSolveAdd() when x = A'b +x */
for (i=0; i<10; i++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = VecSetRandom(s1,rdm);CHKERRQ(ierr);
ierr = VecCopy(s2,s1);CHKERRQ(ierr);
ierr = MatSolveAdd(B,xx,s2,s2);CHKERRQ(ierr);
ierr = MatSolveAdd(A,xx,s1,s1);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatSolveAdd(same) - Norm1=%16.14e Norm2=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
}
/* Test MatSolve() */
for (i=0; i<10; i++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatSolve(B,xx,s2);CHKERRQ(ierr);
ierr = MatSolve(A,xx,s1);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatSolve - Norm1=%16.14e Norm2=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
}
/* Test MatSolveTranspose() */
if (bs < 8) {
for (i=0; i<10; i++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatSolveTranspose(B,xx,s2);CHKERRQ(ierr);
ierr = MatSolveTranspose(A,xx,s1);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatSolveTranspose - Norm1=%16.14e Norm2=%16.14e bs = %D\n",s1norm,s2norm,bs);CHKERRQ(ierr);
}
}
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
ierr = VecDestroy(&yy);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatLUFactorNumeric_SuperLU_DIST(Mat F,Mat A,const MatFactorInfo *info)
{
Mat *tseq,A_seq = NULL;
Mat_SeqAIJ *aa,*bb;
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)(F)->spptr;
PetscErrorCode ierr;
PetscInt M=A->rmap->N,N=A->cmap->N,i,*ai,*aj,*bi,*bj,nz,rstart,*garray,
m=A->rmap->n, colA_start,j,jcol,jB,countA,countB,*bjj,*ajj;
int sinfo; /* SuperLU_Dist info flag is always an int even with long long indices */
PetscMPIInt size;
SuperLUStat_t stat;
double *berr=0;
IS isrow;
Mat F_diag=NULL;
#if defined(PETSC_USE_COMPLEX)
doublecomplex *av, *bv;
#else
double *av, *bv;
#endif
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&size);CHKERRQ(ierr);
if (lu->MatInputMode == GLOBAL) { /* global mat input */
if (size > 1) { /* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
A_seq = *tseq;
ierr = PetscFree(tseq);CHKERRQ(ierr);
aa = (Mat_SeqAIJ*)A_seq->data;
} else {
PetscBool flg;
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&flg);CHKERRQ(ierr);
if (flg) {
Mat_MPIAIJ *At = (Mat_MPIAIJ*)A->data;
A = At->A;
}
aa = (Mat_SeqAIJ*)A->data;
}
/* Convert Petsc NR matrix to SuperLU_DIST NC.
Note: memories of lu->val, col and row are allocated by CompRow_to_CompCol_dist()! */
if (lu->options.Fact != DOFACT) {/* successive numeric factorization, sparsity pattern is reused. */
PetscStackCall("SuperLU_DIST:Destroy_CompCol_Matrix_dist",Destroy_CompCol_Matrix_dist(&lu->A_sup));
if (lu->FactPattern == SamePattern_SameRowPerm) {
lu->options.Fact = SamePattern_SameRowPerm; /* matrix has similar numerical values */
} else { /* lu->FactPattern == SamePattern */
PetscStackCall("SuperLU_DIST:Destroy_LU",Destroy_LU(N, &lu->grid, &lu->LUstruct));
lu->options.Fact = SamePattern;
}
}
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCompRow_to_CompCol_dist",zCompRow_to_CompCol_dist(M,N,aa->nz,(doublecomplex*)aa->a,(int_t*)aa->j,(int_t*)aa->i,&lu->val,&lu->col, &lu->row));
#else
PetscStackCall("SuperLU_DIST:dCompRow_to_CompCol_dist",dCompRow_to_CompCol_dist(M,N,aa->nz,aa->a,(int_t*)aa->j,(int_t*)aa->i,&lu->val, &lu->col, &lu->row));
#endif
/* Create compressed column matrix A_sup. */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCreate_CompCol_Matrix_dist",zCreate_CompCol_Matrix_dist(&lu->A_sup, M, N, aa->nz, lu->val, lu->col, lu->row, SLU_NC, SLU_Z, SLU_GE));
#else
PetscStackCall("SuperLU_DIST:dCreate_CompCol_Matrix_dist",dCreate_CompCol_Matrix_dist(&lu->A_sup, M, N, aa->nz, lu->val, lu->col, lu->row, SLU_NC, SLU_D, SLU_GE));
#endif
} else { /* distributed mat input */
Mat_MPIAIJ *mat = (Mat_MPIAIJ*)A->data;
aa=(Mat_SeqAIJ*)(mat->A)->data;
bb=(Mat_SeqAIJ*)(mat->B)->data;
ai=aa->i; aj=aa->j;
bi=bb->i; bj=bb->j;
#if defined(PETSC_USE_COMPLEX)
av=(doublecomplex*)aa->a;
bv=(doublecomplex*)bb->a;
#else
av=aa->a;
bv=bb->a;
#endif
rstart = A->rmap->rstart;
nz = aa->nz + bb->nz;
garray = mat->garray;
if (lu->options.Fact == DOFACT) { /* first numeric factorization */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zallocateA_dist",zallocateA_dist(m, nz, &lu->val, &lu->col, &lu->row));
#else
PetscStackCall("SuperLU_DIST:dallocateA_dist",dallocateA_dist(m, nz, &lu->val, &lu->col, &lu->row));
#endif
} else { /* successive numeric factorization, sparsity pattern and perm_c are reused. */
/* Destroy_CompRowLoc_Matrix_dist(&lu->A_sup); */ /* this leads to crash! However, see SuperLU_DIST_2.5/EXAMPLE/pzdrive2.c */
if (lu->FactPattern == SamePattern_SameRowPerm) {
lu->options.Fact = SamePattern_SameRowPerm; /* matrix has similar numerical values */
} else {
PetscStackCall("SuperLU_DIST:Destroy_LU",Destroy_LU(N, &lu->grid, &lu->LUstruct)); /* Deallocate storage associated with the L and U matrices. */
lu->options.Fact = SamePattern;
}
}
nz = 0;
for (i=0; i<m; i++) {
lu->row[i] = nz;
countA = ai[i+1] - ai[i];
countB = bi[i+1] - bi[i];
ajj = aj + ai[i]; /* ptr to the beginning of this row */
bjj = bj + bi[i];
/* B part, smaller col index */
colA_start = rstart + ajj[0]; /* the smallest global col index of A */
jB = 0;
for (j=0; j<countB; j++) {
jcol = garray[bjj[j]];
if (jcol > colA_start) {
jB = j;
break;
}
lu->col[nz] = jcol;
lu->val[nz++] = *bv++;
if (j==countB-1) jB = countB;
}
/* A part */
for (j=0; j<countA; j++) {
lu->col[nz] = rstart + ajj[j];
lu->val[nz++] = *av++;
}
/* B part, larger col index */
for (j=jB; j<countB; j++) {
lu->col[nz] = garray[bjj[j]];
lu->val[nz++] = *bv++;
}
}
lu->row[m] = nz;
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCreate_CompRowLoc_Matrix_dist",zCreate_CompRowLoc_Matrix_dist(&lu->A_sup, M, N, nz, m, rstart,lu->val, lu->col, lu->row, SLU_NR_loc, SLU_Z, SLU_GE));
#else
PetscStackCall("SuperLU_DIST:dCreate_CompRowLoc_Matrix_dist",dCreate_CompRowLoc_Matrix_dist(&lu->A_sup, M, N, nz, m, rstart,lu->val, lu->col, lu->row, SLU_NR_loc, SLU_D, SLU_GE));
#endif
}
/* Factor the matrix. */
PetscStackCall("SuperLU_DIST:PStatInit",PStatInit(&stat)); /* Initialize the statistics variables. */
if (lu->MatInputMode == GLOBAL) { /* global mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx_ABglobal",pzgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, M, 0,&lu->grid, &lu->LUstruct, berr, &stat, &sinfo));
#else
PetscStackCall("SuperLU_DIST:pdgssvx_ABglobal",pdgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, M, 0,&lu->grid, &lu->LUstruct, berr, &stat, &sinfo));
#endif
} else { /* distributed mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx",pzgssvx(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, m, 0, &lu->grid,&lu->LUstruct, &lu->SOLVEstruct, berr, &stat, &sinfo));
if (sinfo) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pzgssvx fails, info: %d\n",sinfo);
#else
PetscStackCall("SuperLU_DIST:pdgssvx",pdgssvx(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, m, 0, &lu->grid,&lu->LUstruct, &lu->SOLVEstruct, berr, &stat, &sinfo));
if (sinfo) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pdgssvx fails, info: %d\n",sinfo);
#endif
}
if (lu->MatInputMode == GLOBAL && size > 1) {
ierr = MatDestroy(&A_seq);CHKERRQ(ierr);
}
if (lu->options.PrintStat) {
PStatPrint(&lu->options, &stat, &lu->grid); /* Print the statistics. */
}
PStatFree(&stat);
if (size > 1) {
F_diag = ((Mat_MPIAIJ*)(F)->data)->A;
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
(F)->preallocated = PETSC_TRUE;
lu->options.Fact = FACTORED; /* The factored form of A is supplied. Local option used by this func. only */
PetscFunctionReturn(0);
}
PetscErrorCode KSPDestroy_AGMRES(KSP ksp)
{
PetscErrorCode ierr;
KSP_AGMRES *agmres = (KSP_AGMRES*)ksp->data;
PetscFunctionBegin;
ierr = PetscFree(agmres->hh_origin);CHKERRQ(ierr);
ierr = PetscFree(agmres->nrs);CHKERRQ(ierr);
ierr = PetscFree(agmres->Qloc);CHKERRQ(ierr);
ierr = PetscFree(agmres->Rloc);CHKERRQ(ierr);
ierr = PetscFree(agmres->sgn);CHKERRQ(ierr);
ierr = PetscFree(agmres->tloc);CHKERRQ(ierr);
ierr = PetscFree(agmres->Rshift);CHKERRQ(ierr);
ierr = PetscFree(agmres->Ishift);CHKERRQ(ierr);
ierr = PetscFree(agmres->Scale);CHKERRQ(ierr);
ierr = PetscFree(agmres->wbufptr);CHKERRQ(ierr);
ierr = PetscFree(agmres->tau);CHKERRQ(ierr);
ierr = PetscFree(agmres->work);CHKERRQ(ierr);
ierr = PetscFree(agmres->temp);CHKERRQ(ierr);
ierr = PetscFree(agmres->select);CHKERRQ(ierr);
ierr = PetscFree(agmres->wr);CHKERRQ(ierr);
ierr = PetscFree(agmres->wi);CHKERRQ(ierr);
if (agmres->neig) {
ierr = VecDestroyVecs(MAXKSPSIZE,&agmres->TmpU);CHKERRQ(ierr);
ierr = PetscFree(agmres->perm);CHKERRQ(ierr);
ierr = PetscFree(agmres->MatEigL);CHKERRQ(ierr);
ierr = PetscFree(agmres->MatEigR);CHKERRQ(ierr);
ierr = PetscFree(agmres->Q);CHKERRQ(ierr);
ierr = PetscFree(agmres->Z);CHKERRQ(ierr);
ierr = PetscFree(agmres->beta);CHKERRQ(ierr);
}
ierr = KSPDestroy_DGMRES(ksp);
PetscFunctionReturn(0);
}
PetscErrorCode PCSetUp_MG(PC pc)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
PetscInt i,n = mglevels[0]->levels;
PC cpc;
PetscBool preonly,lu,redundant,cholesky,svd,dump = PETSC_FALSE,opsset,use_amat;
Mat dA,dB;
Vec tvec;
DM *dms;
PetscViewer viewer = 0;
PetscFunctionBegin;
/* FIX: Move this to PCSetFromOptions_MG? */
if (mg->usedmfornumberoflevels) {
PetscInt levels;
ierr = DMGetRefineLevel(pc->dm,&levels);CHKERRQ(ierr);
levels++;
if (levels > n) { /* the problem is now being solved on a finer grid */
ierr = PCMGSetLevels(pc,levels,NULL);CHKERRQ(ierr);
n = levels;
ierr = PCSetFromOptions(pc);CHKERRQ(ierr); /* it is bad to call this here, but otherwise will never be called for the new hierarchy */
mglevels = mg->levels;
}
}
ierr = KSPGetPC(mglevels[0]->smoothd,&cpc);CHKERRQ(ierr);
/* If user did not provide fine grid operators OR operator was not updated since last global KSPSetOperators() */
/* so use those from global PC */
/* Is this what we always want? What if user wants to keep old one? */
ierr = KSPGetOperatorsSet(mglevels[n-1]->smoothd,NULL,&opsset);CHKERRQ(ierr);
if (opsset) {
Mat mmat;
ierr = KSPGetOperators(mglevels[n-1]->smoothd,NULL,&mmat);CHKERRQ(ierr);
if (mmat == pc->pmat) opsset = PETSC_FALSE;
}
if (!opsset) {
ierr = PCGetUseAmat(pc,&use_amat);CHKERRQ(ierr);
if(use_amat){
ierr = PetscInfo(pc,"Using outer operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->mat,pc->pmat);CHKERRQ(ierr);
}
else {
ierr = PetscInfo(pc,"Using matrix (pmat) operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->pmat,pc->pmat);CHKERRQ(ierr);
}
}
/* Skipping this for galerkin==2 (externally managed hierarchy such as ML and GAMG). Cleaner logic here would be great. Wrap ML/GAMG as DMs? */
if (pc->dm && mg->galerkin != 2 && !pc->setupcalled) {
/* construct the interpolation from the DMs */
Mat p;
Vec rscale;
ierr = PetscMalloc1(n,&dms);CHKERRQ(ierr);
dms[n-1] = pc->dm;
/* Separately create them so we do not get DMKSP interference between levels */
for (i=n-2; i>-1; i--) {ierr = DMCoarsen(dms[i+1],MPI_COMM_NULL,&dms[i]);CHKERRQ(ierr);}
for (i=n-2; i>-1; i--) {
DMKSP kdm;
ierr = KSPSetDM(mglevels[i]->smoothd,dms[i]);CHKERRQ(ierr);
if (mg->galerkin) {ierr = KSPSetDMActive(mglevels[i]->smoothd,PETSC_FALSE);CHKERRQ(ierr);}
ierr = DMGetDMKSPWrite(dms[i],&kdm);CHKERRQ(ierr);
/* Ugly hack so that the next KSPSetUp() will use the RHS that we set. A better fix is to change dmActive to take
* a bitwise OR of computing the matrix, RHS, and initial iterate. */
kdm->ops->computerhs = NULL;
kdm->rhsctx = NULL;
if (!mglevels[i+1]->interpolate) {
ierr = DMCreateInterpolation(dms[i],dms[i+1],&p,&rscale);CHKERRQ(ierr);
ierr = PCMGSetInterpolation(pc,i+1,p);CHKERRQ(ierr);
if (rscale) {ierr = PCMGSetRScale(pc,i+1,rscale);CHKERRQ(ierr);}
ierr = VecDestroy(&rscale);CHKERRQ(ierr);
ierr = MatDestroy(&p);CHKERRQ(ierr);
}
}
for (i=n-2; i>-1; i--) {ierr = DMDestroy(&dms[i]);CHKERRQ(ierr);}
ierr = PetscFree(dms);CHKERRQ(ierr);
}
if (pc->dm && !pc->setupcalled) {
/* finest smoother also gets DM but it is not active, independent of whether galerkin==2 */
ierr = KSPSetDM(mglevels[n-1]->smoothd,pc->dm);CHKERRQ(ierr);
ierr = KSPSetDMActive(mglevels[n-1]->smoothd,PETSC_FALSE);CHKERRQ(ierr);
}
if (mg->galerkin == 1) {
Mat B;
/* currently only handle case where mat and pmat are the same on coarser levels */
ierr = KSPGetOperators(mglevels[n-1]->smoothd,&dA,&dB);CHKERRQ(ierr);
if (!pc->setupcalled) {
for (i=n-2; i>-1; i--) {
if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
if (!mglevels[i+1]->interpolate) {
ierr = PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);CHKERRQ(ierr);
}
if (!mglevels[i+1]->restrct) {
ierr = PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);CHKERRQ(ierr);
}
if (mglevels[i+1]->interpolate == mglevels[i+1]->restrct) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
} else {
ierr = MatMatMatMult(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
}
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B);CHKERRQ(ierr);
if (i != n-2) {ierr = PetscObjectDereference((PetscObject)dB);CHKERRQ(ierr);}
dB = B;
}
if (n > 1) {ierr = PetscObjectDereference((PetscObject)dB);CHKERRQ(ierr);}
} else {
for (i=n-2; i>-1; i--) {
if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
if (!mglevels[i+1]->interpolate) {
ierr = PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);CHKERRQ(ierr);
}
if (!mglevels[i+1]->restrct) {
ierr = PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);CHKERRQ(ierr);
}
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&B);CHKERRQ(ierr);
if (mglevels[i+1]->interpolate == mglevels[i+1]->restrct) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);CHKERRQ(ierr);
} else {
ierr = MatMatMatMult(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);CHKERRQ(ierr);
}
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B);CHKERRQ(ierr);
dB = B;
}
}
} else if (!mg->galerkin && pc->dm && pc->dm->x) {
/* need to restrict Jacobian location to coarser meshes for evaluation */
for (i=n-2; i>-1; i--) {
Mat R;
Vec rscale;
if (!mglevels[i]->smoothd->dm->x) {
Vec *vecs;
ierr = KSPCreateVecs(mglevels[i]->smoothd,1,&vecs,0,NULL);CHKERRQ(ierr);
mglevels[i]->smoothd->dm->x = vecs[0];
ierr = PetscFree(vecs);CHKERRQ(ierr);
}
ierr = PCMGGetRestriction(pc,i+1,&R);CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);CHKERRQ(ierr);
ierr = MatRestrict(R,mglevels[i+1]->smoothd->dm->x,mglevels[i]->smoothd->dm->x);CHKERRQ(ierr);
ierr = VecPointwiseMult(mglevels[i]->smoothd->dm->x,mglevels[i]->smoothd->dm->x,rscale);CHKERRQ(ierr);
}
}
if (!mg->galerkin && pc->dm) {
for (i=n-2; i>=0; i--) {
DM dmfine,dmcoarse;
Mat Restrict,Inject;
Vec rscale;
ierr = KSPGetDM(mglevels[i+1]->smoothd,&dmfine);CHKERRQ(ierr);
ierr = KSPGetDM(mglevels[i]->smoothd,&dmcoarse);CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i+1,&Restrict);CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);CHKERRQ(ierr);
Inject = NULL; /* Callback should create it if it needs Injection */
ierr = DMRestrict(dmfine,Restrict,rscale,Inject,dmcoarse);CHKERRQ(ierr);
}
}
if (!pc->setupcalled) {
for (i=0; i<n; i++) {
ierr = KSPSetFromOptions(mglevels[i]->smoothd);CHKERRQ(ierr);
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && (mglevels[i]->smoothu != mglevels[i]->smoothd)) {
ierr = KSPSetFromOptions(mglevels[i]->smoothu);CHKERRQ(ierr);
}
}
for (i=1; i<n; i++) {
ierr = PCMGGetInterpolation(pc,i,&mglevels[i]->interpolate);CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i,&mglevels[i]->restrct);CHKERRQ(ierr);
}
for (i=0; i<n-1; i++) {
if (!mglevels[i]->b) {
Vec *vec;
ierr = KSPCreateVecs(mglevels[i]->smoothd,1,&vec,0,NULL);CHKERRQ(ierr);
ierr = PCMGSetRhs(pc,i,*vec);CHKERRQ(ierr);
ierr = VecDestroy(vec);CHKERRQ(ierr);
ierr = PetscFree(vec);CHKERRQ(ierr);
}
if (!mglevels[i]->r && i) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);CHKERRQ(ierr);
ierr = PCMGSetR(pc,i,tvec);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
}
if (!mglevels[i]->x) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);CHKERRQ(ierr);
ierr = PCMGSetX(pc,i,tvec);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
}
}
if (n != 1 && !mglevels[n-1]->r) {
/* PCMGSetR() on the finest level if user did not supply it */
Vec *vec;
ierr = KSPCreateVecs(mglevels[n-1]->smoothd,1,&vec,0,NULL);CHKERRQ(ierr);
ierr = PCMGSetR(pc,n-1,*vec);CHKERRQ(ierr);
ierr = VecDestroy(vec);CHKERRQ(ierr);
ierr = PetscFree(vec);CHKERRQ(ierr);
}
}
if (pc->dm) {
/* need to tell all the coarser levels to rebuild the matrix using the DM for that level */
for (i=0; i<n-1; i++) {
if (mglevels[i]->smoothd->setupstage != KSP_SETUP_NEW) mglevels[i]->smoothd->setupstage = KSP_SETUP_NEWMATRIX;
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu == mglevels[i]->smoothd || mg->am == PC_MG_FULL || mg->am == PC_MG_KASKADE || mg->cyclesperpcapply > 1){
/* if doing only down then initial guess is zero */
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothd,PETSC_TRUE);CHKERRQ(ierr);
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[i]->smoothd);CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
if (!mglevels[i]->residual) {
Mat mat;
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&mat);CHKERRQ(ierr);
ierr = PCMGSetResidual(pc,i,PCMGResidualDefault,mat);CHKERRQ(ierr);
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && mglevels[i]->smoothu != mglevels[i]->smoothd) {
Mat downmat,downpmat;
/* check if operators have been set for up, if not use down operators to set them */
ierr = KSPGetOperatorsSet(mglevels[i]->smoothu,&opsset,NULL);CHKERRQ(ierr);
if (!opsset) {
ierr = KSPGetOperators(mglevels[i]->smoothd,&downmat,&downpmat);CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[i]->smoothu,downmat,downpmat);CHKERRQ(ierr);
}
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothu,PETSC_TRUE);CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[i]->smoothu);CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
}
}
/*
If coarse solver is not direct method then DO NOT USE preonly
*/
ierr = PetscObjectTypeCompare((PetscObject)mglevels[0]->smoothd,KSPPREONLY,&preonly);CHKERRQ(ierr);
if (preonly) {
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCLU,&lu);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCREDUNDANT,&redundant);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCCHOLESKY,&cholesky);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCSVD,&svd);CHKERRQ(ierr);
if (!lu && !redundant && !cholesky && !svd) {
ierr = KSPSetType(mglevels[0]->smoothd,KSPGMRES);CHKERRQ(ierr);
}
}
if (mglevels[0]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[0]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[0]->smoothd);CHKERRQ(ierr);
if (mglevels[0]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[0]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
/*
Dump the interpolation/restriction matrices plus the
Jacobian/stiffness on each level. This allows MATLAB users to
easily check if the Galerkin condition A_c = R A_f R^T is satisfied.
Only support one or the other at the same time.
*/
#if defined(PETSC_USE_SOCKET_VIEWER)
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_mg_dump_matlab",&dump,NULL);CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_SOCKET_(PetscObjectComm((PetscObject)pc));
dump = PETSC_FALSE;
#endif
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_mg_dump_binary",&dump,NULL);CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_BINARY_(PetscObjectComm((PetscObject)pc));
if (viewer) {
for (i=1; i<n; i++) {
ierr = MatView(mglevels[i]->restrct,viewer);CHKERRQ(ierr);
}
for (i=0; i<n; i++) {
ierr = KSPGetPC(mglevels[i]->smoothd,&pc);CHKERRQ(ierr);
ierr = MatView(pc->mat,viewer);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
/*
spbas_mergesort
mergesort for an array of intergers and an array of associated
reals
on output, icol[0..nnz-1] is increasing;
val[0..nnz-1] has undergone the same permutation as icol
NB: val may be NULL: in that case, only the integers are sorted
*/
PetscErrorCode spbas_mergesort(PetscInt nnz, PetscInt *icol, PetscScalar *val)
{
PetscInt istep; /* Chunk-sizes of already sorted parts of arrays */
PetscInt i, i1, i2; /* Loop counters for (partly) sorted arrays */
PetscInt istart, i1end, i2end; /* start of newly sorted array part, end of both parts */
PetscInt *ialloc; /* Allocated arrays */
PetscScalar *valloc=NULL;
PetscInt *iswap; /* auxiliary pointers for swapping */
PetscScalar *vswap;
PetscInt *ihlp1; /* Pointers to new version of arrays, */
PetscScalar *vhlp1=NULL; /* (arrays under construction) */
PetscInt *ihlp2; /* Pointers to previous version of arrays, */
PetscScalar *vhlp2=NULL;
PetscErrorCode ierr;
ierr = PetscMalloc1(nnz,&ialloc);CHKERRQ(ierr);
ihlp1 = ialloc;
ihlp2 = icol;
if (val) {
ierr = PetscMalloc1(nnz,&valloc);CHKERRQ(ierr);
vhlp1 = valloc;
vhlp2 = val;
}
/* Sorted array chunks are first 1 long, and increase until they are the complete array */
for (istep=1; istep<nnz; istep*=2) {
/*
Combine sorted parts
istart:istart+istep-1 and istart+istep-1:istart+2*istep-1
of ihlp2 and vhlp2
into one sorted part
istart:istart+2*istep-1
of ihlp1 and vhlp1
*/
for (istart=0; istart<nnz; istart+=2*istep) {
/* Set counters and bound array part endings */
i1=istart; i1end = i1+istep; if (i1end>nnz) i1end=nnz;
i2=istart+istep; i2end = i2+istep; if (i2end>nnz) i2end=nnz;
/* Merge the two array parts */
if (val) {
for (i=istart; i<i2end; i++) {
if (i1<i1end && i2<i2end && ihlp2[i1] < ihlp2[i2]) {
ihlp1[i] = ihlp2[i1];
vhlp1[i] = vhlp2[i1];
i1++;
} else if (i2<i2end) {
ihlp1[i] = ihlp2[i2];
vhlp1[i] = vhlp2[i2];
i2++;
} else {
ihlp1[i] = ihlp2[i1];
vhlp1[i] = vhlp2[i1];
i1++;
}
}
} else {
for (i=istart; i<i2end; i++) {
if (i1<i1end && i2<i2end && ihlp2[i1] < ihlp2[i2]) {
ihlp1[i] = ihlp2[i1];
i1++;
} else if (i2<i2end) {
ihlp1[i] = ihlp2[i2];
i2++;
} else {
ihlp1[i] = ihlp2[i1];
i1++;
}
}
}
}
/* Swap the two array sets */
iswap = ihlp2; ihlp2 = ihlp1; ihlp1 = iswap;
vswap = vhlp2; vhlp2 = vhlp1; vhlp1 = vswap;
}
/* Copy one more time in case the sorted arrays are the temporary ones */
if (ihlp2 != icol) {
for (i=0; i<nnz; i++) icol[i] = ihlp2[i];
if (val) {
for (i=0; i<nnz; i++) val[i] = vhlp2[i];
}
}
ierr = PetscFree(ialloc);CHKERRQ(ierr);
if (val) {ierr = PetscFree(valloc);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCNullSpaceAssembleCoarse(PC pc, MatNullSpace* CoarseNullSpace)
{
PC_BDDC *pcbddc = (PC_BDDC*)pc->data;
Mat_IS *matis = (Mat_IS*)pc->pmat->data;
MatNullSpace tempCoarseNullSpace;
const Vec *nsp_vecs;
Vec *coarse_nsp_vecs,local_vec,local_primal_vec;
PetscInt nsp_size,coarse_nsp_size,i;
PetscBool nsp_has_cnst;
PetscReal test_null;
PetscErrorCode ierr;
PetscFunctionBegin;
tempCoarseNullSpace = 0;
coarse_nsp_size = 0;
coarse_nsp_vecs = 0;
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nsp_has_cnst,&nsp_size,&nsp_vecs);CHKERRQ(ierr);
if (pcbddc->coarse_mat) {
ierr = PetscMalloc((nsp_size+1)*sizeof(Vec),&coarse_nsp_vecs);CHKERRQ(ierr);
for (i=0;i<nsp_size+1;i++) {
ierr = VecDuplicate(pcbddc->coarse_vec,&coarse_nsp_vecs[i]);CHKERRQ(ierr);
}
}
ierr = MatGetVecs(pcbddc->ConstraintMatrix,&local_vec,&local_primal_vec);CHKERRQ(ierr);
if (nsp_has_cnst) {
ierr = VecSet(local_vec,1.0);CHKERRQ(ierr);
ierr = MatMult(pcbddc->ConstraintMatrix,local_vec,local_primal_vec);CHKERRQ(ierr);
ierr = PCBDDCScatterCoarseDataBegin(pc,local_primal_vec,pcbddc->coarse_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PCBDDCScatterCoarseDataEnd(pc,local_primal_vec,pcbddc->coarse_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (pcbddc->coarse_mat) {
if (pcbddc->dbg_flag) {
ierr = MatMult(pcbddc->coarse_mat,pcbddc->coarse_vec,pcbddc->coarse_rhs);CHKERRQ(ierr);
ierr = VecNorm(pcbddc->coarse_rhs,NORM_INFINITY,&test_null);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pcbddc->dbg_viewer,"Constant coarse null space error % 1.14e\n",test_null);CHKERRQ(ierr);
}
ierr = VecCopy(pcbddc->coarse_vec,coarse_nsp_vecs[coarse_nsp_size]);CHKERRQ(ierr);
coarse_nsp_size++;
}
}
for (i=0;i<nsp_size;i++) {
ierr = VecScatterBegin(matis->ctx,nsp_vecs[i],local_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->ctx,nsp_vecs[i],local_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = MatMult(pcbddc->ConstraintMatrix,local_vec,local_primal_vec);CHKERRQ(ierr);
ierr = PCBDDCScatterCoarseDataBegin(pc,local_primal_vec,pcbddc->coarse_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PCBDDCScatterCoarseDataEnd(pc,local_primal_vec,pcbddc->coarse_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (pcbddc->coarse_mat) {
if (pcbddc->dbg_flag) {
ierr = MatMult(pcbddc->coarse_mat,pcbddc->coarse_vec,pcbddc->coarse_rhs);CHKERRQ(ierr);
ierr = VecNorm(pcbddc->coarse_rhs,NORM_2,&test_null);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pcbddc->dbg_viewer,"Vec %d coarse null space error % 1.14e\n",i,test_null);CHKERRQ(ierr);
}
ierr = VecCopy(pcbddc->coarse_vec,coarse_nsp_vecs[coarse_nsp_size]);CHKERRQ(ierr);
coarse_nsp_size++;
}
}
if (coarse_nsp_size > 0) {
ierr = PCBDDCOrthonormalizeVecs(coarse_nsp_size,coarse_nsp_vecs);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)(pcbddc->coarse_mat)),PETSC_FALSE,coarse_nsp_size,coarse_nsp_vecs,&tempCoarseNullSpace);CHKERRQ(ierr);
for (i=0;i<nsp_size+1;i++) {
ierr = VecDestroy(&coarse_nsp_vecs[i]);CHKERRQ(ierr);
}
}
ierr = PetscFree(coarse_nsp_vecs);CHKERRQ(ierr);
ierr = VecDestroy(&local_vec);CHKERRQ(ierr);
ierr = VecDestroy(&local_primal_vec);CHKERRQ(ierr);
*CoarseNullSpace = tempCoarseNullSpace;
PetscFunctionReturn(0);
}
static PetscErrorCode VecAssemblyBegin_MPI_BTS(Vec X)
{
Vec_MPI *x = (Vec_MPI*)X->data;
PetscErrorCode ierr;
MPI_Comm comm;
PetscInt i,j,jb,bs;
PetscFunctionBegin;
if (X->stash.donotstash) PetscFunctionReturn(0);
ierr = PetscObjectGetComm((PetscObject)X,&comm);CHKERRQ(ierr);
ierr = VecGetBlockSize(X,&bs);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
{
InsertMode addv;
ierr = MPIU_Allreduce((PetscEnum*)&X->stash.insertmode,(PetscEnum*)&addv,1,MPIU_ENUM,MPI_BOR,comm);CHKERRQ(ierr);
if (addv == (ADD_VALUES|INSERT_VALUES)) SETERRQ(comm,PETSC_ERR_ARG_NOTSAMETYPE,"Some processors inserted values while others added");
}
#endif
X->bstash.insertmode = X->stash.insertmode; /* Block stash implicitly tracks InsertMode of scalar stash */
ierr = VecStashSortCompress_Private(&X->stash);CHKERRQ(ierr);
ierr = VecStashSortCompress_Private(&X->bstash);CHKERRQ(ierr);
if (!x->sendranks) {
PetscMPIInt nowners,bnowners,*owners,*bowners;
PetscInt ntmp;
ierr = VecStashGetOwnerList_Private(&X->stash,X->map,&nowners,&owners);CHKERRQ(ierr);
ierr = VecStashGetOwnerList_Private(&X->bstash,X->map,&bnowners,&bowners);CHKERRQ(ierr);
ierr = PetscMergeMPIIntArray(nowners,owners,bnowners,bowners,&ntmp,&x->sendranks);CHKERRQ(ierr);
x->nsendranks = ntmp;
ierr = PetscFree(owners);CHKERRQ(ierr);
ierr = PetscFree(bowners);CHKERRQ(ierr);
ierr = PetscMalloc1(x->nsendranks,&x->sendhdr);CHKERRQ(ierr);
ierr = PetscCalloc1(x->nsendranks,&x->sendptrs);CHKERRQ(ierr);
}
for (i=0,j=0,jb=0; i<x->nsendranks; i++) {
PetscMPIInt rank = x->sendranks[i];
x->sendhdr[i].insertmode = X->stash.insertmode;
/* Initialize pointers for non-empty stashes the first time around. Subsequent assemblies with
* VEC_SUBSET_OFF_PROC_ENTRIES will leave the old pointers (dangling because the stash has been collected) when
* there is nothing new to send, so that size-zero messages get sent instead. */
x->sendhdr[i].count = 0;
if (X->stash.n) {
x->sendptrs[i].ints = &X->stash.idx[j];
x->sendptrs[i].scalars = &X->stash.array[j];
for ( ; j<X->stash.n && X->stash.idx[j] < X->map->range[rank+1]; j++) x->sendhdr[i].count++;
}
x->sendhdr[i].bcount = 0;
if (X->bstash.n) {
x->sendptrs[i].intb = &X->bstash.idx[jb];
x->sendptrs[i].scalarb = &X->bstash.array[jb*bs];
for ( ; jb<X->bstash.n && X->bstash.idx[jb]*bs < X->map->range[rank+1]; jb++) x->sendhdr[i].bcount++;
}
}
if (!x->segrecvint) {ierr = PetscSegBufferCreate(sizeof(PetscInt),1000,&x->segrecvint);CHKERRQ(ierr);}
if (!x->segrecvscalar) {ierr = PetscSegBufferCreate(sizeof(PetscScalar),1000,&x->segrecvscalar);CHKERRQ(ierr);}
if (!x->segrecvframe) {ierr = PetscSegBufferCreate(sizeof(VecAssemblyFrame),50,&x->segrecvframe);CHKERRQ(ierr);}
if (x->recvhdr) { /* VEC_SUBSET_OFF_PROC_ENTRIES and this is not the first assembly */
PetscMPIInt tag[4];
if (!x->assembly_subset) SETERRQ(comm,PETSC_ERR_PLIB,"Attempt to reuse rendezvous when not VEC_SUBSET_OFF_PROC_ENTRIES");
for (i=0; i<4; i++) {ierr = PetscCommGetNewTag(comm,&tag[i]);CHKERRQ(ierr);}
for (i=0; i<x->nsendranks; i++) {
ierr = VecAssemblySend_MPI_Private(comm,tag,i,x->sendranks[i],x->sendhdr+i,x->sendreqs+4*i,X);CHKERRQ(ierr);
}
for (i=0; i<x->nrecvranks; i++) {
ierr = VecAssemblyRecv_MPI_Private(comm,tag,x->recvranks[i],x->recvhdr+i,x->recvreqs+4*i,X);CHKERRQ(ierr);
}
x->use_status = PETSC_TRUE;
} else { /* First time */
ierr = PetscCommBuildTwoSidedFReq(comm,3,MPIU_INT,x->nsendranks,x->sendranks,(PetscInt*)x->sendhdr,&x->nrecvranks,&x->recvranks,&x->recvhdr,4,&x->sendreqs,&x->recvreqs,VecAssemblySend_MPI_Private,VecAssemblyRecv_MPI_Private,X);CHKERRQ(ierr);
x->use_status = PETSC_FALSE;
}
{
PetscInt nstash,reallocs;
ierr = VecStashGetInfo_Private(&X->stash,&nstash,&reallocs);CHKERRQ(ierr);
ierr = PetscInfo2(X,"Stash has %D entries, uses %D mallocs.\n",nstash,reallocs);CHKERRQ(ierr);
ierr = VecStashGetInfo_Private(&X->bstash,&nstash,&reallocs);CHKERRQ(ierr);
ierr = PetscInfo2(X,"Block-Stash has %D entries, uses %D mallocs.\n",nstash,reallocs);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCNullSpaceAssembleCorrection(PC pc,IS local_dofs)
{
PC_BDDC *pcbddc = (PC_BDDC*)pc->data;
PC_IS *pcis = (PC_IS*)pc->data;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
KSP *local_ksp;
PC newpc;
NullSpaceCorrection_ctx shell_ctx;
Mat local_mat,local_pmat,small_mat,inv_small_mat;
MatStructure local_mat_struct;
Vec work1,work2;
const Vec *nullvecs;
VecScatter scatter_ctx;
IS is_aux;
MatFactorInfo matinfo;
PetscScalar *basis_mat,*Kbasis_mat,*array,*array_mat;
PetscScalar one = 1.0,zero = 0.0, m_one = -1.0;
PetscInt basis_dofs,basis_size,nnsp_size,i,k,n_I,n_R;
PetscBool nnsp_has_cnst;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Infer the local solver */
ierr = ISGetSize(local_dofs,&basis_dofs);CHKERRQ(ierr);
ierr = VecGetSize(pcis->vec1_D,&n_I);CHKERRQ(ierr);
ierr = VecGetSize(pcbddc->vec1_R,&n_R);CHKERRQ(ierr);
if (basis_dofs == n_I) {
/* Dirichlet solver */
local_ksp = &pcbddc->ksp_D;
} else if (basis_dofs == n_R) {
/* Neumann solver */
local_ksp = &pcbddc->ksp_R;
} else {
SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error in %s: unknown local IS size %d. n_I=%d, n_R=%d)\n",__FUNCT__,basis_dofs,n_I,n_R);
}
ierr = KSPGetOperators(*local_ksp,&local_mat,&local_pmat,&local_mat_struct);CHKERRQ(ierr);
/* Get null space vecs */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nnsp_has_cnst,&nnsp_size,&nullvecs);CHKERRQ(ierr);
basis_size = nnsp_size;
if (nnsp_has_cnst) {
basis_size++;
}
/* Create shell ctx */
ierr = PetscMalloc(sizeof(*shell_ctx),&shell_ctx);CHKERRQ(ierr);
/* Create work vectors in shell context */
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_small_1,basis_size,basis_size);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_small_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_small_1,&shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_full_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_full_1,basis_dofs,basis_dofs);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_full_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&shell_ctx->work_full_2);CHKERRQ(ierr);
/* Allocate workspace */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->basis_mat );CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->Kbasis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Restrict local null space on selected dofs (Dirichlet or Neumann)
and compute matrices N and K*N */
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,local_dofs,work1,(IS)0,&scatter_ctx);CHKERRQ(ierr);
for (k=0;k<nnsp_size;k++) {
ierr = VecScatterBegin(matis->ctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->ctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
if (nnsp_has_cnst) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecSet(work1,one);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter_ctx);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Assemble another Mat object in shell context */
ierr = MatTransposeMatMult(shell_ctx->basis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&small_mat);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&matinfo);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,basis_size,0,1,&is_aux);CHKERRQ(ierr);
ierr = MatLUFactor(small_mat,is_aux,is_aux,&matinfo);CHKERRQ(ierr);
ierr = ISDestroy(&is_aux);CHKERRQ(ierr);
ierr = PetscMalloc(basis_size*basis_size*sizeof(PetscScalar),&array_mat);CHKERRQ(ierr);
for (k=0;k<basis_size;k++) {
ierr = VecSet(shell_ctx->work_small_1,zero);CHKERRQ(ierr);
ierr = VecSetValue(shell_ctx->work_small_1,k,one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecAssemblyEnd(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = MatSolve(small_mat,shell_ctx->work_small_1,shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecGetArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
for (i=0;i<basis_size;i++) {
array_mat[i*basis_size+k]=array[i];
}
ierr = VecRestoreArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_size,basis_size,array_mat,&inv_small_mat);CHKERRQ(ierr);
ierr = MatMatMult(shell_ctx->basis_mat,inv_small_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&shell_ctx->Lbasis_mat);CHKERRQ(ierr);
ierr = PetscFree(array_mat);CHKERRQ(ierr);
ierr = MatDestroy(&inv_small_mat);CHKERRQ(ierr);
ierr = MatDestroy(&small_mat);CHKERRQ(ierr);
ierr = MatScale(shell_ctx->Kbasis_mat,m_one);CHKERRQ(ierr);
/* Rebuild local PC */
ierr = KSPGetPC(*local_ksp,&shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PCCreate(PETSC_COMM_SELF,&newpc);CHKERRQ(ierr);
ierr = PCSetOperators(newpc,local_mat,local_mat,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = PCSetType(newpc,PCSHELL);CHKERRQ(ierr);
ierr = PCShellSetContext(newpc,shell_ctx);CHKERRQ(ierr);
ierr = PCShellSetApply(newpc,PCBDDCApplyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCShellSetDestroy(newpc,PCBDDCDestroyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCSetUp(newpc);CHKERRQ(ierr);
ierr = KSPSetPC(*local_ksp,newpc);CHKERRQ(ierr);
ierr = PCDestroy(&newpc);CHKERRQ(ierr);
ierr = KSPSetUp(*local_ksp);CHKERRQ(ierr);
/* test */
/* TODO: this cause a deadlock when doing multilevel */
#if 0
if (pcbddc->dbg_flag) {
KSP check_ksp;
PC check_pc;
Mat test_mat;
Vec work3;
PetscViewer viewer=pcbddc->dbg_viewer;
PetscReal test_err,lambda_min,lambda_max;
PetscBool setsym,issym=PETSC_FALSE;
ierr = KSPGetPC(*local_ksp,&check_pc);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work3);CHKERRQ(ierr);
ierr = VecSetRandom(shell_ctx->work_small_1,NULL);CHKERRQ(ierr);
ierr = MatMult(shell_ctx->basis_mat,shell_ctx->work_small_1,work1);CHKERRQ(ierr);
ierr = VecCopy(work1,work2);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work3);CHKERRQ(ierr);
ierr = PCApply(check_pc,work3,work1);CHKERRQ(ierr);
ierr = VecAXPY(work1,m_one,work2);CHKERRQ(ierr);
ierr = VecNorm(work1,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d error for nullspace correction for ",PetscGlobalRank);
if (basis_dofs == n_I) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Dirichlet ");
} else {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Neumann ");
}
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"solver is :%1.14e\n",test_err);
ierr = MatTransposeMatMult(shell_ctx->Lbasis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&test_mat);CHKERRQ(ierr);
ierr = MatShift(test_mat,one);CHKERRQ(ierr);
ierr = MatNorm(test_mat,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = MatDestroy(&test_mat);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d error for nullspace matrices is :%1.14e\n",PetscGlobalRank,test_err);
/* Create ksp object suitable for extreme eigenvalues' estimation */
ierr = KSPCreate(PETSC_COMM_SELF,&check_ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(check_ksp,local_mat,local_mat,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetTolerances(check_ksp,1.e-8,1.e-8,PETSC_DEFAULT,basis_dofs);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(check_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatIsSymmetricKnown(pc->pmat,&setsym,&issym);CHKERRQ(ierr);
if (issym) {
ierr = KSPSetType(check_ksp,KSPCG);CHKERRQ(ierr);
}
ierr = KSPSetPC(check_ksp,check_pc);CHKERRQ(ierr);
ierr = KSPSetUp(check_ksp);CHKERRQ(ierr);
ierr = VecSetRandom(work1,NULL);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = KSPSolve(check_ksp,work2,work2);CHKERRQ(ierr);
ierr = VecAXPY(work2,m_one,work1);CHKERRQ(ierr);
ierr = VecNorm(work2,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(check_ksp,&lambda_max,&lambda_min);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(check_ksp,&k);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d error for adapted KSP %1.14e (it %d, eigs %1.6e %1.6e)\n",PetscGlobalRank,test_err,k,lambda_min,lambda_max);
ierr = KSPDestroy(&check_ksp);CHKERRQ(ierr);
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecDestroy(&work3);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
}
#endif
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCNullSpaceAdaptGlobal(PC pc)
{
PC_IS* pcis = (PC_IS*) (pc->data);
PC_BDDC* pcbddc = (PC_BDDC*)(pc->data);
KSP inv_change;
PC pc_change;
const Vec *nsp_vecs;
Vec *new_nsp_vecs;
PetscInt i,nsp_size,new_nsp_size,start_new;
PetscBool nsp_has_cnst;
MatNullSpace new_nsp;
PetscErrorCode ierr;
MPI_Comm comm;
PetscFunctionBegin;
/* create KSP for change of basis */
ierr = KSPCreate(PETSC_COMM_SELF,&inv_change);CHKERRQ(ierr);
ierr = KSPSetOperators(inv_change,pcbddc->ChangeOfBasisMatrix,pcbddc->ChangeOfBasisMatrix,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetType(inv_change,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(inv_change,&pc_change);CHKERRQ(ierr);
ierr = PCSetType(pc_change,PCLU);CHKERRQ(ierr);
ierr = KSPSetUp(inv_change);CHKERRQ(ierr);
/* get nullspace and transform it */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nsp_has_cnst,&nsp_size,&nsp_vecs);CHKERRQ(ierr);
new_nsp_size = nsp_size;
if (nsp_has_cnst) {
new_nsp_size++;
}
ierr = PetscMalloc(new_nsp_size*sizeof(Vec),&new_nsp_vecs);CHKERRQ(ierr);
for (i=0;i<new_nsp_size;i++) {
ierr = VecDuplicate(pcis->vec1_global,&new_nsp_vecs[i]);CHKERRQ(ierr);
}
start_new = 0;
if (nsp_has_cnst) {
start_new = 1;
ierr = VecSet(new_nsp_vecs[0],1.0);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_B,1.0);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,pcis->vec1_B,pcis->vec1_B);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[0],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[0],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
for (i=0;i<nsp_size;i++) {
ierr = VecCopy(nsp_vecs[i],new_nsp_vecs[i+start_new]);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,nsp_vecs[i],pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,nsp_vecs[i],pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,pcis->vec1_B,pcis->vec1_B);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[i+start_new],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[i+start_new],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
ierr = PCBDDCOrthonormalizeVecs(new_nsp_size,new_nsp_vecs);CHKERRQ(ierr);
#if 0
PetscBool nsp_t=PETSC_FALSE;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("Original Null Space test: %d\n",nsp_t);
Mat temp_mat;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("Original Null Space, mat changed test: %d\n",nsp_t);
{
PetscReal test_norm;
for (i=0;i<new_nsp_size;i++) {
ierr = MatMult(pc->pmat,new_nsp_vecs[i],pcis->vec1_global);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_2,&test_norm);CHKERRQ(ierr);
if (test_norm > 1.e-12) {
printf("------------ERROR VEC %d------------------\n",i);
ierr = VecView(pcis->vec1_global,PETSC_VIEWER_STDOUT_WORLD);
printf("------------------------------------------\n");
}
}
}
#endif
ierr = KSPDestroy(&inv_change);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(comm,PETSC_FALSE,new_nsp_size,new_nsp_vecs,&new_nsp);CHKERRQ(ierr);
ierr = PCBDDCSetNullSpace(pc,new_nsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&new_nsp);CHKERRQ(ierr);
#if 0
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("New Null Space, mat changed: %d\n",nsp_t);
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("New Null Space, mat original: %d\n",nsp_t);
#endif
for (i=0;i<new_nsp_size;i++) {
ierr = VecDestroy(&new_nsp_vecs[i]);CHKERRQ(ierr);
}
ierr = PetscFree(new_nsp_vecs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt *indices,n;
const PetscInt *nindices;
PetscMPIInt rank;
IS is;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/*
Create an index set with 5 entries. Each processor creates
its own index set with its own list of integers.
*/
ierr = PetscMalloc(5*sizeof(PetscInt),&indices);CHKERRQ(ierr);
indices[0] = rank + 1;
indices[1] = rank + 2;
indices[2] = rank + 3;
indices[3] = rank + 4;
indices[4] = rank + 5;
ierr = ISCreateGeneral(PETSC_COMM_SELF,5,indices,&is);CHKERRQ(ierr);
/*
Note that ISCreateGeneral() has made a copy of the indices
so we may (and generally should) free indices[]
*/
ierr = PetscFree(indices);CHKERRQ(ierr);
/*
Print the index set to stdout
*/
ierr = ISView(is,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Get the number of indices in the set
*/
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
/*
Get the indices in the index set
*/
ierr = ISGetIndices(is,&nindices);CHKERRQ(ierr);
/*
Now any code that needs access to the list of integers
has access to it here through indices[].
*/
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] First index %D\n",rank,nindices[0]);CHKERRQ(ierr);
/*
Once we no longer need access to the indices they should
returned to the system
*/
ierr = ISRestoreIndices(is,&nindices);CHKERRQ(ierr);
/*
One should destroy any PETSc object once one is completely
done with it.
*/
ierr = ISDestroy(is);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
/*@
DMPlexCreateExodus - Create a DMPlex mesh from an ExodusII file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. exoid - The ExodusII id associated with a exodus file and obtained using ex_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Level: beginner
.keywords: mesh,ExodusII
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateExodus(MPI_Comm comm, PetscInt exoid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_EXODUSII)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from ex_get_init() */
char title[PETSC_MAX_PATH_LEN+1];
int dim = 0, numVertices = 0, numCells = 0;
int num_cs = 0, num_vs = 0, num_fs = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_EXODUSII)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open EXODUS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
ierr = PetscMemzero(title,(PETSC_MAX_PATH_LEN+1)*sizeof(char));CHKERRQ(ierr);
ierr = ex_get_init(exoid, title, &dim, &numVertices, &numCells, &num_cs, &num_vs, &num_fs);
if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ExodusII ex_get_init() failed with error code %D\n",ierr);
if (!num_cs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Exodus file does not contain any cell set\n");
}
ierr = MPI_Bcast(title, PETSC_MAX_PATH_LEN+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, title);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read cell sets information */
if (!rank) {
PetscInt *cone;
int c, cs, c_loc, v, v_loc;
/* Read from ex_get_elem_blk_ids() */
int *cs_id;
/* Read from ex_get_elem_block() */
char buffer[PETSC_MAX_PATH_LEN+1];
int num_cell_in_set, num_vertex_per_cell, num_attr;
/* Read from ex_get_elem_conn() */
int *cs_connect;
/* Get cell sets IDs */
ierr = PetscMalloc1(num_cs, &cs_id);CHKERRQ(ierr);
ierr = ex_get_elem_blk_ids(exoid, cs_id);CHKERRQ(ierr);
/* Read the cell set connectivity table and build mesh topology
EXO standard requires that cells in cell sets be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
for (c_loc = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, num_vertex_per_cell);CHKERRQ(ierr);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc2(num_vertex_per_cell*num_cell_in_set,&cs_connect,num_vertex_per_cell,&cone);CHKERRQ(ierr);
ierr = ex_get_elem_conn(exoid, cs_id[cs], cs_connect);CHKERRQ(ierr);
/* EXO uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
for (v_loc = 0; v_loc < num_vertex_per_cell; ++v_loc, ++v) {
cone[v_loc] = cs_connect[v]+numCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (num_vertex_per_cell == 4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (num_vertex_per_cell == 8) {
PetscInt tmp = cone[1];
cone[1] = cone[3];
cone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMSetLabelValue(*dm, "Cell Sets", c, cs_id[cs]);CHKERRQ(ierr);
}
ierr = PetscFree2(cs_connect,cone);CHKERRQ(ierr);
}
ierr = PetscFree(cs_id);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain Cell Sets label */
{
DMLabel label;
ierr = DMRemoveLabel(*dm, "Cell Sets", &label);CHKERRQ(ierr);
if (label) {ierr = DMAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Create vertex set label */
if (!rank && (num_vs > 0)) {
int vs, v;
/* Read from ex_get_node_set_ids() */
int *vs_id;
/* Read from ex_get_node_set_param() */
int num_vertex_in_set, num_attr;
/* Read from ex_get_node_set() */
int *vs_vertex_list;
/* Get vertex set ids */
ierr = PetscMalloc1(num_vs, &vs_id);CHKERRQ(ierr);
ierr = ex_get_node_set_ids(exoid, vs_id);CHKERRQ(ierr);
for (vs = 0; vs < num_vs; ++vs) {
ierr = ex_get_node_set_param(exoid, vs_id[vs], &num_vertex_in_set, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc1(num_vertex_in_set, &vs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_node_set(exoid, vs_id[vs], vs_vertex_list);CHKERRQ(ierr);
for (v = 0; v < num_vertex_in_set; ++v) {
ierr = DMSetLabelValue(*dm, "Vertex Sets", vs_vertex_list[v]+numCells-1, vs_id[vs]);CHKERRQ(ierr);
}
ierr = PetscFree(vs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(vs_id);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
float *x, *y, *z;
ierr = PetscMalloc3(numVertices,&x,numVertices,&y,numVertices,&z);CHKERRQ(ierr);
ierr = ex_get_coord(exoid, x, y, z);CHKERRQ(ierr);
if (dim > 0) {
for (v = 0; v < numVertices; ++v) coords[v*dim+0] = x[v];
}
if (dim > 1) {
for (v = 0; v < numVertices; ++v) coords[v*dim+1] = y[v];
}
if (dim > 2) {
for (v = 0; v < numVertices; ++v) coords[v*dim+2] = z[v];
}
ierr = PetscFree3(x,y,z);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Create side set label */
if (!rank && interpolate && (num_fs > 0)) {
int fs, f, voff;
/* Read from ex_get_side_set_ids() */
int *fs_id;
/* Read from ex_get_side_set_param() */
int num_side_in_set, num_dist_fact_in_set;
/* Read from ex_get_side_set_node_list() */
int *fs_vertex_count_list, *fs_vertex_list;
/* Get side set ids */
ierr = PetscMalloc1(num_fs, &fs_id);CHKERRQ(ierr);
ierr = ex_get_side_set_ids(exoid, fs_id);CHKERRQ(ierr);
for (fs = 0; fs < num_fs; ++fs) {
ierr = ex_get_side_set_param(exoid, fs_id[fs], &num_side_in_set, &num_dist_fact_in_set);CHKERRQ(ierr);
ierr = PetscMalloc2(num_side_in_set,&fs_vertex_count_list,num_side_in_set*4,&fs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_side_set_node_list(exoid, fs_id[fs], fs_vertex_count_list, fs_vertex_list);CHKERRQ(ierr);
for (f = 0, voff = 0; f < num_side_in_set; ++f) {
const PetscInt *faces = NULL;
PetscInt faceSize = fs_vertex_count_list[f], numFaces;
PetscInt faceVertices[4], v;
if (faceSize > 4) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "ExodusII side cannot have %d > 4 vertices", faceSize);
for (v = 0; v < faceSize; ++v, ++voff) {
faceVertices[v] = fs_vertex_list[voff]+numCells-1;
}
ierr = DMPlexGetFullJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Invalid ExodusII side %d in set %d maps to %d faces", f, fs, numFaces);
ierr = DMSetLabelValue(*dm, "Face Sets", faces[0], fs_id[fs]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
}
ierr = PetscFree2(fs_vertex_count_list,fs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(fs_id);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires ExodusII support. Reconfigure using --download-exodusii");
#endif
PetscFunctionReturn(0);
}
PetscErrorCode MatCholeskyFactorNumeric_SeqSBAIJ_4_NaturalOrdering(Mat C,Mat A,const MatFactorInfo *info)
{
Mat_SeqSBAIJ *a = (Mat_SeqSBAIJ*)A->data,*b = (Mat_SeqSBAIJ*)C->data;
PetscErrorCode ierr;
PetscInt i,j,mbs=a->mbs,*bi=b->i,*bj=b->j;
PetscInt *ai,*aj,k,k1,jmin,jmax,*jl,*il,vj,nexti,ili;
MatScalar *ba = b->a,*aa,*ap,*dk,*uik;
MatScalar *u,*diag,*rtmp,*rtmp_ptr;
PetscBool pivotinblocks = b->pivotinblocks;
PetscReal shift = info->shiftamount;
PetscFunctionBegin;
/* initialization */
ierr = PetscMalloc(16*mbs*sizeof(MatScalar),&rtmp);CHKERRQ(ierr);
ierr = PetscMemzero(rtmp,16*mbs*sizeof(MatScalar));CHKERRQ(ierr);
ierr = PetscMalloc2(mbs,PetscInt,&il,mbs,PetscInt,&jl);CHKERRQ(ierr);
for (i=0; i<mbs; i++) {
jl[i] = mbs; il[0] = 0;
}
ierr = PetscMalloc2(16,MatScalar,&dk,16,MatScalar,&uik);CHKERRQ(ierr);
ai = a->i; aj = a->j; aa = a->a;
/* for each row k */
for (k = 0; k<mbs; k++) {
/*initialize k-th row with elements nonzero in row k of A */
jmin = ai[k]; jmax = ai[k+1];
if (jmin < jmax) {
ap = aa + jmin*16;
for (j = jmin; j < jmax; j++) {
vj = aj[j]; /* block col. index */
rtmp_ptr = rtmp + vj*16;
for (i=0; i<16; i++) *rtmp_ptr++ = *ap++;
}
}
/* modify k-th row by adding in those rows i with U(i,k) != 0 */
ierr = PetscMemcpy(dk,rtmp+k*16,16*sizeof(MatScalar));CHKERRQ(ierr);
i = jl[k]; /* first row to be added to k_th row */
while (i < mbs) {
nexti = jl[i]; /* next row to be added to k_th row */
/* compute multiplier */
ili = il[i]; /* index of first nonzero element in U(i,k:bms-1) */
/* uik = -inv(Di)*U_bar(i,k) */
diag = ba + i*16;
u = ba + ili*16;
uik[0] = -(diag[0]*u[0] + diag[4]*u[1] + diag[8]*u[2] + diag[12]*u[3]);
uik[1] = -(diag[1]*u[0] + diag[5]*u[1] + diag[9]*u[2] + diag[13]*u[3]);
uik[2] = -(diag[2]*u[0] + diag[6]*u[1] + diag[10]*u[2]+ diag[14]*u[3]);
uik[3] = -(diag[3]*u[0] + diag[7]*u[1] + diag[11]*u[2]+ diag[15]*u[3]);
uik[4] = -(diag[0]*u[4] + diag[4]*u[5] + diag[8]*u[6] + diag[12]*u[7]);
uik[5] = -(diag[1]*u[4] + diag[5]*u[5] + diag[9]*u[6] + diag[13]*u[7]);
uik[6] = -(diag[2]*u[4] + diag[6]*u[5] + diag[10]*u[6]+ diag[14]*u[7]);
uik[7] = -(diag[3]*u[4] + diag[7]*u[5] + diag[11]*u[6]+ diag[15]*u[7]);
uik[8] = -(diag[0]*u[8] + diag[4]*u[9] + diag[8]*u[10] + diag[12]*u[11]);
uik[9] = -(diag[1]*u[8] + diag[5]*u[9] + diag[9]*u[10] + diag[13]*u[11]);
uik[10]= -(diag[2]*u[8] + diag[6]*u[9] + diag[10]*u[10]+ diag[14]*u[11]);
uik[11]= -(diag[3]*u[8] + diag[7]*u[9] + diag[11]*u[10]+ diag[15]*u[11]);
uik[12]= -(diag[0]*u[12] + diag[4]*u[13] + diag[8]*u[14] + diag[12]*u[15]);
uik[13]= -(diag[1]*u[12] + diag[5]*u[13] + diag[9]*u[14] + diag[13]*u[15]);
uik[14]= -(diag[2]*u[12] + diag[6]*u[13] + diag[10]*u[14]+ diag[14]*u[15]);
uik[15]= -(diag[3]*u[12] + diag[7]*u[13] + diag[11]*u[14]+ diag[15]*u[15]);
/* update D(k) += -U(i,k)^T * U_bar(i,k) */
dk[0] += uik[0]*u[0] + uik[1]*u[1] + uik[2]*u[2] + uik[3]*u[3];
dk[1] += uik[4]*u[0] + uik[5]*u[1] + uik[6]*u[2] + uik[7]*u[3];
dk[2] += uik[8]*u[0] + uik[9]*u[1] + uik[10]*u[2]+ uik[11]*u[3];
dk[3] += uik[12]*u[0]+ uik[13]*u[1]+ uik[14]*u[2]+ uik[15]*u[3];
dk[4] += uik[0]*u[4] + uik[1]*u[5] + uik[2]*u[6] + uik[3]*u[7];
dk[5] += uik[4]*u[4] + uik[5]*u[5] + uik[6]*u[6] + uik[7]*u[7];
dk[6] += uik[8]*u[4] + uik[9]*u[5] + uik[10]*u[6]+ uik[11]*u[7];
dk[7] += uik[12]*u[4]+ uik[13]*u[5]+ uik[14]*u[6]+ uik[15]*u[7];
dk[8] += uik[0]*u[8] + uik[1]*u[9] + uik[2]*u[10] + uik[3]*u[11];
dk[9] += uik[4]*u[8] + uik[5]*u[9] + uik[6]*u[10] + uik[7]*u[11];
dk[10]+= uik[8]*u[8] + uik[9]*u[9] + uik[10]*u[10]+ uik[11]*u[11];
dk[11]+= uik[12]*u[8]+ uik[13]*u[9]+ uik[14]*u[10]+ uik[15]*u[11];
dk[12]+= uik[0]*u[12] + uik[1]*u[13] + uik[2]*u[14] + uik[3]*u[15];
dk[13]+= uik[4]*u[12] + uik[5]*u[13] + uik[6]*u[14] + uik[7]*u[15];
dk[14]+= uik[8]*u[12] + uik[9]*u[13] + uik[10]*u[14]+ uik[11]*u[15];
dk[15]+= uik[12]*u[12]+ uik[13]*u[13]+ uik[14]*u[14]+ uik[15]*u[15];
ierr = PetscLogFlops(64.0*4.0);CHKERRQ(ierr);
/* update -U(i,k) */
ierr = PetscMemcpy(ba+ili*16,uik,16*sizeof(MatScalar));CHKERRQ(ierr);
/* add multiple of row i to k-th row ... */
jmin = ili + 1; jmax = bi[i+1];
if (jmin < jmax) {
for (j=jmin; j<jmax; j++) {
/* rtmp += -U(i,k)^T * U_bar(i,j) */
rtmp_ptr = rtmp + bj[j]*16;
u = ba + j*16;
rtmp_ptr[0] += uik[0]*u[0] + uik[1]*u[1] + uik[2]*u[2] + uik[3]*u[3];
rtmp_ptr[1] += uik[4]*u[0] + uik[5]*u[1] + uik[6]*u[2] + uik[7]*u[3];
rtmp_ptr[2] += uik[8]*u[0] + uik[9]*u[1] + uik[10]*u[2]+ uik[11]*u[3];
rtmp_ptr[3] += uik[12]*u[0]+ uik[13]*u[1]+ uik[14]*u[2]+ uik[15]*u[3];
rtmp_ptr[4] += uik[0]*u[4] + uik[1]*u[5] + uik[2]*u[6] + uik[3]*u[7];
rtmp_ptr[5] += uik[4]*u[4] + uik[5]*u[5] + uik[6]*u[6] + uik[7]*u[7];
rtmp_ptr[6] += uik[8]*u[4] + uik[9]*u[5] + uik[10]*u[6]+ uik[11]*u[7];
rtmp_ptr[7] += uik[12]*u[4]+ uik[13]*u[5]+ uik[14]*u[6]+ uik[15]*u[7];
rtmp_ptr[8] += uik[0]*u[8] + uik[1]*u[9] + uik[2]*u[10] + uik[3]*u[11];
rtmp_ptr[9] += uik[4]*u[8] + uik[5]*u[9] + uik[6]*u[10] + uik[7]*u[11];
rtmp_ptr[10]+= uik[8]*u[8] + uik[9]*u[9] + uik[10]*u[10]+ uik[11]*u[11];
rtmp_ptr[11]+= uik[12]*u[8]+ uik[13]*u[9]+ uik[14]*u[10]+ uik[15]*u[11];
rtmp_ptr[12]+= uik[0]*u[12] + uik[1]*u[13] + uik[2]*u[14] + uik[3]*u[15];
rtmp_ptr[13]+= uik[4]*u[12] + uik[5]*u[13] + uik[6]*u[14] + uik[7]*u[15];
rtmp_ptr[14]+= uik[8]*u[12] + uik[9]*u[13] + uik[10]*u[14]+ uik[11]*u[15];
rtmp_ptr[15]+= uik[12]*u[12]+ uik[13]*u[13]+ uik[14]*u[14]+ uik[15]*u[15];
}
ierr = PetscLogFlops(2.0*64.0*(jmax-jmin));CHKERRQ(ierr);
/* ... add i to row list for next nonzero entry */
il[i] = jmin; /* update il(i) in column k+1, ... mbs-1 */
j = bj[jmin];
jl[i] = jl[j]; jl[j] = i; /* update jl */
}
i = nexti;
}
/* save nonzero entries in k-th row of U ... */
/* invert diagonal block */
diag = ba+k*16;
ierr = PetscMemcpy(diag,dk,16*sizeof(MatScalar));CHKERRQ(ierr);
if (pivotinblocks) {
ierr = PetscKernel_A_gets_inverse_A_4(diag,shift);CHKERRQ(ierr);
} else {
ierr = PetscKernel_A_gets_inverse_A_4_nopivot(diag);CHKERRQ(ierr);
}
jmin = bi[k]; jmax = bi[k+1];
if (jmin < jmax) {
for (j=jmin; j<jmax; j++) {
vj = bj[j]; /* block col. index of U */
u = ba + j*16;
rtmp_ptr = rtmp + vj*16;
for (k1=0; k1<16; k1++) {
*u++ = *rtmp_ptr;
*rtmp_ptr++ = 0.0;
}
}
/* ... add k to row list for first nonzero entry in k-th row */
il[k] = jmin;
i = bj[jmin];
jl[k] = jl[i]; jl[i] = k;
}
}
ierr = PetscFree(rtmp);CHKERRQ(ierr);
ierr = PetscFree2(il,jl);CHKERRQ(ierr);
ierr = PetscFree2(dk,uik);CHKERRQ(ierr);
C->ops->solve = MatSolve_SeqSBAIJ_4_NaturalOrdering_inplace;
C->ops->solvetranspose = MatSolve_SeqSBAIJ_4_NaturalOrdering_inplace;
C->ops->forwardsolve = MatForwardSolve_SeqSBAIJ_4_NaturalOrdering_inplace;
C->ops->backwardsolve = MatBackwardSolve_SeqSBAIJ_4_NaturalOrdering_inplace;
C->assembled = PETSC_TRUE;
C->preallocated = PETSC_TRUE;
ierr = PetscLogFlops(1.3333*64*b->mbs);CHKERRQ(ierr); /* from inverting diagonal blocks */
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt n = 5,i,*blks,bs = 1,m = 2;
PetscScalar value;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
PetscViewer sviewer;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,size*bs*n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
/* create two index sets */
if (rank < size-1) m = n + 2;
else m = n;
ierr = PetscMalloc1(m,&blks);CHKERRQ(ierr);
blks[0] = n*rank;
for (i=1; i<m; i++) blks[i] = blks[i-1] + 1;
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,m,blks,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = PetscFree(blks);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,bs*m,&y);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,bs*m,0,1,&is2);CHKERRQ(ierr);
/* each processor inserts the entire vector */
/* this is redundant but tests assembly */
for (i=0; i<bs*n*size; i++) {
value = (PetscScalar) i;
ierr = VecSetValues(x,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"----\n");CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(y,sviewer);CHKERRQ(ierr); fflush(stdout);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt N0=50,N1=20,N=N0*N1,DIM;
PetscRandom rdm;
PetscScalar a;
PetscReal enorm;
Vec x,y,z;
PetscBool view=PETSC_FALSE,use_interface=PETSC_TRUE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
#if !defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires complex numbers");
#endif
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "FFTW Options", "ex143");CHKERRQ(ierr);
ierr = PetscOptionsBool("-vec_view draw", "View the vectors", "ex143", view, &view, NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-use_FFTW_interface", "Use PETSc-FFTW interface", "ex143",use_interface, &use_interface, NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-use_FFTW_interface",&use_interface,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD, &rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
if (!use_interface) {
/* Use mpi FFTW without PETSc-FFTW interface, 2D case only */
/*---------------------------------------------------------*/
fftw_plan fplan,bplan;
fftw_complex *data_in,*data_out,*data_out2;
ptrdiff_t alloc_local,local_n0,local_0_start;
DIM = 2;
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Use FFTW without PETSc-FFTW interface, DIM %D\n",DIM);CHKERRQ(ierr);
}
fftw_mpi_init();
N = N0*N1;
alloc_local = fftw_mpi_local_size_2d(N0,N1,PETSC_COMM_WORLD,&local_n0,&local_0_start);
data_in = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
data_out = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
data_out2 = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_in,&x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Real Space vector");CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_out,&y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "Frequency space vector");CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_out2,&z);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) z, "Reconstructed vector");CHKERRQ(ierr);
fplan = fftw_mpi_plan_dft_2d(N0,N1,data_in,data_out,PETSC_COMM_WORLD,FFTW_FORWARD,FFTW_ESTIMATE);
bplan = fftw_mpi_plan_dft_2d(N0,N1,data_out,data_out2,PETSC_COMM_WORLD,FFTW_BACKWARD,FFTW_ESTIMATE);
ierr = VecSetRandom(x, rdm);CHKERRQ(ierr);
if (view) {ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
fftw_execute(fplan);
if (view) {ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
fftw_execute(bplan);
/* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
a = 1.0/(PetscReal)N;
ierr = VecScale(z,a);CHKERRQ(ierr);
if (view) {ierr = VecView(z, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
ierr = VecAXPY(z,-1.0,x);CHKERRQ(ierr);
ierr = VecNorm(z,NORM_1,&enorm);CHKERRQ(ierr);
if (enorm > 1.e-11 && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF," Error norm of |x - z| %g\n",(double)enorm);CHKERRQ(ierr);
}
/* Free spaces */
fftw_destroy_plan(fplan);
fftw_destroy_plan(bplan);
fftw_free(data_in); ierr = VecDestroy(&x);CHKERRQ(ierr);
fftw_free(data_out); ierr = VecDestroy(&y);CHKERRQ(ierr);
fftw_free(data_out2);ierr = VecDestroy(&z);CHKERRQ(ierr);
} else {
/* Use PETSc-FFTW interface */
/*-------------------------------------------*/
PetscInt i,*dim,k;
Mat A;
N=1;
for (i=1; i<5; i++) {
DIM = i;
ierr = PetscMalloc1(i,&dim);CHKERRQ(ierr);
for (k=0; k<i; k++) {
dim[k]=30;
}
N *= dim[i-1];
/* Create FFTW object */
if (!rank) printf("Use PETSc-FFTW interface...%d-DIM: %d\n",(int)DIM,(int)N);
ierr = MatCreateFFT(PETSC_COMM_WORLD,DIM,dim,MATFFTW,&A);CHKERRQ(ierr);
/* Create vectors that are compatible with parallel layout of A - must call MatGetVecs()! */
ierr = MatGetVecsFFTW(A,&x,&y,&z);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Real space vector");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "Frequency space vector");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) z, "Reconstructed vector");CHKERRQ(ierr);
/* Set values of space vector x */
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
if (view) {ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
/* Apply FFTW_FORWARD and FFTW_BACKWARD */
ierr = MatMult(A,x,y);CHKERRQ(ierr);
if (view) {ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
ierr = MatMultTranspose(A,y,z);CHKERRQ(ierr);
/* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
a = 1.0/(PetscReal)N;
ierr = VecScale(z,a);CHKERRQ(ierr);
if (view) {ierr = VecView(z,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
ierr = VecAXPY(z,-1.0,x);CHKERRQ(ierr);
ierr = VecNorm(z,NORM_1,&enorm);CHKERRQ(ierr);
if (enorm > 1.e-9 && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF," Error norm of |x - z| %e\n",enorm);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&z);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree(dim);CHKERRQ(ierr);
}
}
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@C
PCMGSetLevels - Sets the number of levels to use with MG.
Must be called before any other MG routine.
Logically Collective on PC
Input Parameters:
+ pc - the preconditioner context
. levels - the number of levels
- comms - optional communicators for each level; this is to allow solving the coarser problems
on smaller sets of processors. Use NULL_OBJECT for default in Fortran
Level: intermediate
Notes:
If the number of levels is one then the multigrid uses the -mg_levels prefix
for setting the level options rather than the -mg_coarse prefix.
.keywords: MG, set, levels, multigrid
.seealso: PCMGSetType(), PCMGGetLevels()
@*/
PetscErrorCode PCMGSetLevels(PC pc,PetscInt levels,MPI_Comm *comms)
{
PetscErrorCode ierr;
PC_MG *mg = (PC_MG*)pc->data;
MPI_Comm comm;
PC_MG_Levels **mglevels = mg->levels;
PetscInt i;
PetscMPIInt size;
const char *prefix;
PC ipc;
PetscInt n;
PetscFunctionBegin;
PetscValidHeaderSpecific(pc,PC_CLASSID,1);
PetscValidLogicalCollectiveInt(pc,levels,2);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
if (mg->nlevels == levels) PetscFunctionReturn(0);
if (mglevels) {
/* changing the number of levels so free up the previous stuff */
ierr = PCReset_MG(pc);CHKERRQ(ierr);
n = mglevels[0]->levels;
for (i=0; i<n; i++) {
if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
ierr = KSPDestroy(&mglevels[i]->smoothd);CHKERRQ(ierr);
}
ierr = KSPDestroy(&mglevels[i]->smoothu);CHKERRQ(ierr);
ierr = PetscFree(mglevels[i]);CHKERRQ(ierr);
}
ierr = PetscFree(mg->levels);CHKERRQ(ierr);
}
mg->nlevels = levels;
ierr = PetscMalloc1(levels,&mglevels);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,levels*(sizeof(PC_MG*)));CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
mg->stageApply = 0;
for (i=0; i<levels; i++) {
ierr = PetscNewLog(pc,&mglevels[i]);CHKERRQ(ierr);
mglevels[i]->level = i;
mglevels[i]->levels = levels;
mglevels[i]->cycles = PC_MG_CYCLE_V;
mg->default_smoothu = 2;
mg->default_smoothd = 2;
mglevels[i]->eventsmoothsetup = 0;
mglevels[i]->eventsmoothsolve = 0;
mglevels[i]->eventresidual = 0;
mglevels[i]->eventinterprestrict = 0;
if (comms) comm = comms[i];
ierr = KSPCreate(comm,&mglevels[i]->smoothd);CHKERRQ(ierr);
ierr = KSPSetType(mglevels[i]->smoothd,KSPCHEBYSHEV);CHKERRQ(ierr);
ierr = KSPSetConvergenceTest(mglevels[i]->smoothd,KSPConvergedSkip,NULL,NULL);CHKERRQ(ierr);
ierr = KSPSetNormType(mglevels[i]->smoothd,KSP_NORM_NONE);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[i]->smoothd,&ipc);CHKERRQ(ierr);
ierr = PCSetType(ipc,PCSOR);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)mglevels[i]->smoothd,(PetscObject)pc,levels-i);CHKERRQ(ierr);
ierr = KSPSetTolerances(mglevels[i]->smoothd,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT, i ? mg->default_smoothd : 1);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(mglevels[i]->smoothd,prefix);CHKERRQ(ierr);
/* do special stuff for coarse grid */
if (!i && levels > 1) {
ierr = KSPAppendOptionsPrefix(mglevels[0]->smoothd,"mg_coarse_");CHKERRQ(ierr);
/* coarse solve is (redundant) LU by default; set shifttype NONZERO to avoid annoying zero-pivot in LU preconditioner */
ierr = KSPSetType(mglevels[0]->smoothd,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[0]->smoothd,&ipc);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size > 1) {
KSP innerksp;
PC innerpc;
ierr = PCSetType(ipc,PCREDUNDANT);CHKERRQ(ierr);
ierr = PCRedundantGetKSP(ipc,&innerksp);CHKERRQ(ierr);
ierr = KSPGetPC(innerksp,&innerpc);CHKERRQ(ierr);
ierr = PCFactorSetShiftType(innerpc,MAT_SHIFT_INBLOCKS);CHKERRQ(ierr);
} else {
ierr = PCSetType(ipc,PCLU);CHKERRQ(ierr);
ierr = PCFactorSetShiftType(ipc,MAT_SHIFT_INBLOCKS);CHKERRQ(ierr);
}
} else {
char tprefix[128];
sprintf(tprefix,"mg_levels_%d_",(int)i);
ierr = KSPAppendOptionsPrefix(mglevels[i]->smoothd,tprefix);CHKERRQ(ierr);
}
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)mglevels[i]->smoothd);CHKERRQ(ierr);
mglevels[i]->smoothu = mglevels[i]->smoothd;
mg->rtol = 0.0;
mg->abstol = 0.0;
mg->dtol = 0.0;
mg->ttol = 0.0;
mg->cyclesperpcapply = 1;
}
mg->am = PC_MG_MULTIPLICATIVE;
mg->levels = mglevels;
pc->ops->applyrichardson = PCApplyRichardson_MG;
PetscFunctionReturn(0);
}
PetscErrorCode MatGetMultiProcBlock_MPIAIJ(Mat mat, MPI_Comm subComm, MatReuse scall,Mat *subMat)
{
PetscErrorCode ierr;
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)mat->data;
Mat_SeqAIJ *aijB = (Mat_SeqAIJ*)aij->B->data;
PetscMPIInt commRank,subCommSize,subCommRank;
PetscMPIInt *commRankMap,subRank,rank,commsize;
PetscInt *garrayCMap,col,i,j,*nnz,newRow,newCol;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&commsize);CHKERRQ(ierr);
ierr = MPI_Comm_size(subComm,&subCommSize);CHKERRQ(ierr);
/* create subMat object with the relavent layout */
if (scall == MAT_INITIAL_MATRIX) {
ierr = MatCreate(subComm,subMat);CHKERRQ(ierr);
ierr = MatSetType(*subMat,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatSetSizes(*subMat,mat->rmap->n,mat->cmap->n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetBlockSizes(*subMat,mat->rmap->bs,mat->cmap->bs);CHKERRQ(ierr);
/* need to setup rmap and cmap before Preallocation */
ierr = PetscLayoutSetBlockSize((*subMat)->rmap,mat->rmap->bs);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize((*subMat)->cmap,mat->cmap->bs);CHKERRQ(ierr);
ierr = PetscLayoutSetUp((*subMat)->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp((*subMat)->cmap);CHKERRQ(ierr);
}
/* create a map of comm_rank from subComm to comm - should commRankMap and garrayCMap be kept for reused? */
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)mat),&commRank);CHKERRQ(ierr);
ierr = MPI_Comm_rank(subComm,&subCommRank);CHKERRQ(ierr);
ierr = PetscMalloc1(subCommSize,&commRankMap);CHKERRQ(ierr);
ierr = MPI_Allgather(&commRank,1,MPI_INT,commRankMap,1,MPI_INT,subComm);CHKERRQ(ierr);
/* Traverse garray and identify column indices [of offdiag mat] that
should be discarded. For the ones not discarded, store the newCol+1
value in garrayCMap */
ierr = PetscCalloc1(aij->B->cmap->n,&garrayCMap);CHKERRQ(ierr);
for (i=0; i<aij->B->cmap->n; i++) {
col = aij->garray[i];
for (subRank=0; subRank<subCommSize; subRank++) {
rank = commRankMap[subRank];
if ((col >= mat->cmap->range[rank]) && (col < mat->cmap->range[rank+1])) {
garrayCMap[i] = (*subMat)->cmap->range[subRank] + col - mat->cmap->range[rank]+1;
break;
}
}
}
if (scall == MAT_INITIAL_MATRIX) {
/* Now compute preallocation for the offdiag mat */
ierr = PetscCalloc1(aij->B->rmap->n,&nnz);CHKERRQ(ierr);
for (i=0; i<aij->B->rmap->n; i++) {
for (j=aijB->i[i]; j<aijB->i[i+1]; j++) {
if (garrayCMap[aijB->j[j]]) nnz[i]++;
}
}
ierr = MatMPIAIJSetPreallocation(*(subMat),0,NULL,0,nnz);CHKERRQ(ierr);
/* reuse diag block with the new submat */
ierr = MatDestroy(&((Mat_MPIAIJ*)((*subMat)->data))->A);CHKERRQ(ierr);
((Mat_MPIAIJ*)((*subMat)->data))->A = aij->A;
ierr = PetscObjectReference((PetscObject)aij->A);CHKERRQ(ierr);
} else if (((Mat_MPIAIJ*)(*subMat)->data)->A != aij->A) {
PetscObject obj = (PetscObject)((Mat_MPIAIJ*)((*subMat)->data))->A;
ierr = PetscObjectReference((PetscObject)obj);CHKERRQ(ierr);
((Mat_MPIAIJ*)((*subMat)->data))->A = aij->A;
ierr = PetscObjectReference((PetscObject)aij->A);CHKERRQ(ierr);
}
/* Now traverse aij->B and insert values into subMat */
for (i=0; i<aij->B->rmap->n; i++) {
newRow = (*subMat)->rmap->range[subCommRank] + i;
for (j=aijB->i[i]; j<aijB->i[i+1]; j++) {
newCol = garrayCMap[aijB->j[j]];
if (newCol) {
newCol--; /* remove the increment */
ierr = MatSetValues(*subMat,1,&newRow,1,&newCol,(aijB->a+j),INSERT_VALUES);CHKERRQ(ierr);
}
}
}
/* assemble the submat */
ierr = MatAssemblyBegin(*subMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*subMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* deallocate temporary data */
ierr = PetscFree(commRankMap);CHKERRQ(ierr);
ierr = PetscFree(garrayCMap);CHKERRQ(ierr);
if (scall == MAT_INITIAL_MATRIX) {
ierr = PetscFree(nnz);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
spbas_compress_pattern:
calculate a compressed sparseness pattern for a sparseness pattern
given in compressed row storage. The compressed sparseness pattern may
require (much) less memory.
*/
PetscErrorCode spbas_compress_pattern(PetscInt *irow_in, PetscInt *icol_in, PetscInt nrows, PetscInt ncols, PetscInt col_idx_type, spbas_matrix *B,PetscReal *mem_reduction)
{
PetscInt nnz = irow_in[nrows];
size_t mem_orig = (nrows + nnz) * sizeof(PetscInt);
size_t mem_compressed;
PetscErrorCode ierr;
PetscInt *isort;
PetscInt *icols;
PetscInt row_nnz;
PetscInt *ipoint;
PetscBool *used;
PetscInt ptr;
PetscInt i,j;
const PetscBool no_values = PETSC_FALSE;
PetscFunctionBegin;
/* Allocate the structure of the new matrix */
B->nrows = nrows;
B->ncols = ncols;
B->nnz = nnz;
B->col_idx_type = col_idx_type;
B->block_data = PETSC_TRUE;
ierr = spbas_allocate_pattern(B, no_values);CHKERRQ(ierr);
/* When using an offset array, set it */
if (col_idx_type==SPBAS_OFFSET_ARRAY) {
for (i=0; i<nrows; i++) B->icol0[i] = icol_in[irow_in[i]];
}
/* Allocate the ordering for the rows */
ierr = PetscMalloc1(nrows,&isort);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows,&ipoint);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows,&used);CHKERRQ(ierr);
/* Initialize the sorting */
ierr = PetscMemzero((void*) used, nrows*sizeof(PetscBool));CHKERRQ(ierr);
for (i = 0; i<nrows; i++) {
B->row_nnz[i] = irow_in[i+1]-irow_in[i];
isort[i] = i;
ipoint[i] = i;
}
/* Sort the rows so that identical columns will be next to each other */
ierr = spbas_mergesort_icols(nrows, irow_in, icol_in, col_idx_type, isort);CHKERRQ(ierr);
ierr = PetscInfo(NULL,"Rows have been sorted for patterns\n");CHKERRQ(ierr);
/* Replace identical rows with the first one in the list */
for (i=1; i<nrows; i++) {
if (spbas_row_order_icol(isort[i-1], isort[i], irow_in, icol_in, col_idx_type) == 0) {
ipoint[isort[i]] = ipoint[isort[i-1]];
}
}
/* Collect the rows which are used*/
for (i=0; i<nrows; i++) used[ipoint[i]] = PETSC_TRUE;
/* Calculate needed memory */
B->n_alloc_icol = 0;
for (i=0; i<nrows; i++) {
if (used[i]) B->n_alloc_icol += B->row_nnz[i];
}
ierr = PetscMalloc1(B->n_alloc_icol,&B->alloc_icol);CHKERRQ(ierr);
/* Fill in the diagonal offsets for the rows which store their own data */
ptr = 0;
for (i=0; i<B->nrows; i++) {
if (used[i]) {
B->icols[i] = &B->alloc_icol[ptr];
icols = &icol_in[irow_in[i]];
row_nnz = B->row_nnz[i];
if (col_idx_type == SPBAS_COLUMN_NUMBERS) {
for (j=0; j<row_nnz; j++) {
B->icols[i][j] = icols[j];
}
} else if (col_idx_type == SPBAS_DIAGONAL_OFFSETS) {
for (j=0; j<row_nnz; j++) {
B->icols[i][j] = icols[j]-i;
}
} else if (col_idx_type == SPBAS_OFFSET_ARRAY) {
for (j=0; j<row_nnz; j++) {
B->icols[i][j] = icols[j]-icols[0];
}
}
ptr += B->row_nnz[i];
}
}
/* Point to the right places for all data */
for (i=0; i<nrows; i++) {
B->icols[i] = B->icols[ipoint[i]];
}
ierr = PetscInfo(NULL,"Row patterns have been compressed\n");CHKERRQ(ierr);
ierr = PetscInfo1(NULL," (%g nonzeros per row)\n", (double) ((PetscReal) nnz / (PetscReal) nrows));CHKERRQ(ierr);
ierr=PetscFree(isort);CHKERRQ(ierr);
ierr=PetscFree(used);CHKERRQ(ierr);
ierr=PetscFree(ipoint);CHKERRQ(ierr);
mem_compressed = spbas_memory_requirement(*B);
*mem_reduction = 100.0 * (PetscReal)(mem_orig-mem_compressed)/ (PetscReal) mem_orig;
PetscFunctionReturn(0);
}
/*@
PetscDrawBarDraw - Redraws a bar graph.
Collective, but ignored by all processors except processor 0 in PetscDrawBar
Input Parameter:
. bar - The bar graph context
Level: intermediate
@*/
PetscErrorCode PetscDrawBarDraw(PetscDrawBar bar)
{
PetscDraw draw;
PetscBool isnull;
PetscReal xmin,xmax,ymin,ymax,*values,binLeft,binRight;
PetscInt numValues,i,bcolor,color,idx,*perm,nplot;
PetscMPIInt rank;
PetscErrorCode ierr;
char **labels;
PetscFunctionBegin;
if (!bar) PetscFunctionReturn(0);
PetscValidHeaderSpecific(bar,PETSC_DRAWBAR_CLASSID,1);
draw = bar->win;
ierr = PetscDrawIsNull(draw,&isnull);
CHKERRQ(ierr);
if (isnull) PetscFunctionReturn(0);
if (bar->numBins < 1) PetscFunctionReturn(0);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)bar),&rank);
CHKERRQ(ierr);
color = bar->color;
if (color == PETSC_DRAW_ROTATE) bcolor = 2;
else bcolor = color;
numValues = bar->numBins;
values = bar->values;
if (bar->ymin == bar->ymax) {
/* user has not set bounds on bars so set them based on the data */
ymin = PETSC_MAX_REAL;
ymax = PETSC_MIN_REAL;
for (i=0; i<numValues; i++) {
ymin = PetscMin(ymin,values[i]);
ymax = PetscMax(ymax,values[i]);
}
} else {
ymin = bar->ymin;
ymax = bar->ymax;
}
nplot = numValues; /* number of points to actually plot; if some are lower than requested tolerance */
xmin = 0.0;
xmax = nplot;
labels = bar->labels;
if (bar->sort) {
ierr = PetscMalloc1(numValues,&perm);
CHKERRQ(ierr);
for (i=0; i<numValues; i++) perm[i] = i;
ierr = PetscSortRealWithPermutation(numValues,values,perm);
CHKERRQ(ierr);
if (bar->sorttolerance) {
for (i=0; i<numValues; i++) {
if (values[perm[numValues - i - 1]] < bar->sorttolerance) {
nplot = i;
break;
}
}
}
}
ierr = PetscDrawCheckResizedWindow(draw);
CHKERRQ(ierr);
ierr = PetscDrawSynchronizedClear(draw);
CHKERRQ(ierr);
ierr = PetscDrawCollectiveBegin(draw);
CHKERRQ(ierr);
ierr = PetscDrawAxisSetLimits(bar->axis,xmin,xmax,ymin,ymax);
CHKERRQ(ierr);
ierr = PetscDrawAxisDraw(bar->axis);
CHKERRQ(ierr);
if (!rank) { /* Draw bins */
for (i=0; i<nplot; i++) {
idx = (bar->sort ? perm[numValues - i - 1] : i);
binLeft = xmin + i;
binRight = xmin + i + 1;
ierr = PetscDrawRectangle(draw,binLeft,ymin,binRight,values[idx],bcolor,bcolor,bcolor,bcolor);
CHKERRQ(ierr);
ierr = PetscDrawLine(draw,binLeft,ymin,binLeft,values[idx],PETSC_DRAW_BLACK);
CHKERRQ(ierr);
ierr = PetscDrawLine(draw,binRight,ymin,binRight,values[idx],PETSC_DRAW_BLACK);
CHKERRQ(ierr);
ierr = PetscDrawLine(draw,binLeft,values[idx],binRight,values[idx],PETSC_DRAW_BLACK);
CHKERRQ(ierr);
if (labels) {
PetscReal h;
ierr = PetscDrawStringGetSize(draw,NULL,&h);
CHKERRQ(ierr);
ierr = PetscDrawStringCentered(draw,.5*(binLeft+binRight),ymin - 1.2*h,bcolor,labels[idx]);
CHKERRQ(ierr);
}
if (color == PETSC_DRAW_ROTATE) bcolor++;
if (bcolor > PETSC_DRAW_BASIC_COLORS-1) bcolor = 2;
}
}
if (bar->sort) {
ierr = PetscFree(perm);
CHKERRQ(ierr);
}
ierr = PetscDrawCollectiveEnd(draw);
CHKERRQ(ierr);
ierr = PetscDrawSynchronizedFlush(draw);
CHKERRQ(ierr);
ierr = PetscDrawPause(draw);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
spbas_power
Calculate sparseness patterns for incomplete Cholesky decompositions
of a given order: (almost) all nonzeros of the matrix^(order+1) which
are inside the band width are found and stored in the output sparseness
pattern.
*/
PetscErrorCode spbas_power(spbas_matrix in_matrix,PetscInt power, spbas_matrix * result)
{
spbas_matrix retval;
PetscInt nrows = in_matrix.nrows;
PetscInt ncols = in_matrix.ncols;
PetscInt i, j, kend;
PetscInt nnz, inz;
PetscInt *iwork;
PetscInt marker;
PetscInt maxmrk=0;
PetscErrorCode ierr;
PetscFunctionBegin;
if (in_matrix.col_idx_type != SPBAS_DIAGONAL_OFFSETS) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS,"must have diagonal offsets in pattern\n");
if (ncols != nrows) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Dimension error\n");
if (in_matrix.values) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Input array must be sparseness pattern (no values)");
if (power<=0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Power must be 1 or up");
/* Copy input values*/
retval.nrows = ncols;
retval.ncols = nrows;
retval.nnz = 0;
retval.col_idx_type = SPBAS_DIAGONAL_OFFSETS;
retval.block_data = PETSC_FALSE;
/* Allocate sparseness pattern */
ierr = spbas_allocate_pattern(&retval, in_matrix.values ? PETSC_TRUE : PETSC_FALSE);CHKERRQ(ierr);
/* Allocate marker array */
ierr = PetscMalloc1(nrows, &iwork);CHKERRQ(ierr);
/* Erase the pattern for this row */
ierr = PetscMemzero((void*) iwork, retval.nrows*sizeof(PetscInt));CHKERRQ(ierr);
/* Calculate marker values */
marker = 1; for (i=1; i<power; i++) marker*=2;
for (i=0; i<nrows; i++) {
/* Calculate the pattern for each row */
nnz = in_matrix.row_nnz[i];
kend = i+in_matrix.icols[i][nnz-1];
if (maxmrk<=kend) maxmrk=kend+1;
ierr = spbas_mark_row_power(iwork, i, &in_matrix, marker, i, maxmrk);CHKERRQ(ierr);
/* Count the columns*/
nnz = 0;
for (j=i; j<maxmrk; j++) nnz+= (iwork[j]!=0);
/* Allocate the column indices */
retval.row_nnz[i] = nnz;
ierr = PetscMalloc1(nnz,&retval.icols[i]);CHKERRQ(ierr);
/* Administrate the column indices */
inz = 0;
for (j=i; j<maxmrk; j++) {
if (iwork[j]) {
retval.icols[i][inz] = j-i;
inz++;
iwork[j]=0;
}
}
retval.nnz += nnz;
};
ierr = PetscFree(iwork);CHKERRQ(ierr);
*result = retval;
PetscFunctionReturn(0);
}
/*@C
PetscOptionsGetViewer - Gets a viewer appropriate for the type indicated by the user
Collective on MPI_Comm
Input Parameters:
+ comm - the communicator to own the viewer
. pre - the string to prepend to the name or NULL
- name - the option one is seeking
Output Parameter:
+ viewer - the viewer, pass NULL if not needed
. format - the PetscViewerFormat requested by the user, pass NULL if not needed
- set - PETSC_TRUE if found, else PETSC_FALSE
Level: intermediate
Notes: If no value is provided ascii:stdout is used
$ ascii[:[filename][:[format][:append]]] defaults to stdout - format can be one of ascii_info, ascii_info_detail, or ascii_matlab,
for example ascii::ascii_info prints just the information about the object not all details
unless :append is given filename opens in write mode, overwriting what was already there
$ binary[:[filename][:[format][:append]]] defaults to the file binaryoutput
$ draw[:drawtype] for example, draw:tikz or draw:x
$ socket[:port] defaults to the standard output port
$ saws[:communicatorname] publishes object to the Scientific Application Webserver (SAWs)
Use PetscViewerDestroy() after using the viewer, otherwise a memory leak will occur
.seealso: PetscOptionsGetReal(), PetscOptionsHasName(), PetscOptionsGetString(),
PetscOptionsGetIntArray(), PetscOptionsGetRealArray(), PetscOptionsBool()
PetscOptionsInt(), PetscOptionsString(), PetscOptionsReal(), PetscOptionsBool(),
PetscOptionsName(), PetscOptionsBegin(), PetscOptionsEnd(), PetscOptionsHead(),
PetscOptionsStringArray(),PetscOptionsRealArray(), PetscOptionsScalar(),
PetscOptionsBoolGroupBegin(), PetscOptionsBoolGroup(), PetscOptionsBoolGroupEnd(),
PetscOptionsFList(), PetscOptionsEList()
@*/
PetscErrorCode PetscOptionsGetViewer(MPI_Comm comm,const char pre[],const char name[],PetscViewer *viewer,PetscViewerFormat *format,PetscBool *set)
{
char *value;
PetscErrorCode ierr;
PetscBool flag,hashelp;
PetscFunctionBegin;
PetscValidCharPointer(name,3);
ierr = PetscOptionsHasName(NULL,"-help",&hashelp);CHKERRQ(ierr);
if (hashelp) {
ierr = (*PetscHelpPrintf)(comm," -%s%s ascii[:[filename][:[format][:append]]]: %s (%s)\n",pre ? pre : "",name+1,"Triggers display of a PETSc object to screen or ASCII file","PetscOptionsGetViewer");CHKERRQ(ierr);
ierr = (*PetscHelpPrintf)(comm," -%s%s binary[:[filename][:[format][:append]]]: %s (%s)\n",pre ? pre : "",name+1,"Triggers saving of a PETSc object to a binary file","PetscOptionsGetViewer");CHKERRQ(ierr);
ierr = (*PetscHelpPrintf)(comm," -%s%s draw[:drawtype]: %s (%s)\n",pre ? pre : "",name+1,"Triggers drawing of a PETSc object","PetscOptionsGetViewer");CHKERRQ(ierr);
ierr = (*PetscHelpPrintf)(comm," -%s%s socket[:port]: %s (%s)\n",pre ? pre : "",name+1,"Triggers push of a PETSc object to a Unix socket","PetscOptionsGetViewer");CHKERRQ(ierr);
ierr = (*PetscHelpPrintf)(comm," -%s%s saws[:communicatorname]: %s (%s)\n",pre ? pre : "",name+1,"Triggers publishing of a PETSc object to SAWs","PetscOptionsGetViewer");CHKERRQ(ierr);
}
if (format) *format = PETSC_VIEWER_DEFAULT;
if (set) *set = PETSC_FALSE;
ierr = PetscOptionsFindPair_Private(pre,name,&value,&flag);CHKERRQ(ierr);
if (flag) {
if (set) *set = PETSC_TRUE;
if (!value) {
if (viewer) {
ierr = PetscViewerASCIIGetStdout(comm,viewer);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)*viewer);CHKERRQ(ierr);
}
} else {
char *loc0_vtype,*loc1_fname,*loc2_fmt = NULL,*loc3_fmode = NULL;
PetscInt cnt;
const char *viewers[] = {PETSCVIEWERASCII,PETSCVIEWERBINARY,PETSCVIEWERDRAW,PETSCVIEWERSOCKET,PETSCVIEWERMATLAB,PETSCVIEWERSAWS,PETSCVIEWERVTK,PETSCVIEWERHDF5,0};
ierr = PetscStrallocpy(value,&loc0_vtype);CHKERRQ(ierr);
ierr = PetscStrchr(loc0_vtype,':',&loc1_fname);CHKERRQ(ierr);
if (loc1_fname) {
*loc1_fname++ = 0;
ierr = PetscStrchr(loc1_fname,':',&loc2_fmt);CHKERRQ(ierr);
}
if (loc2_fmt) {
*loc2_fmt++ = 0;
ierr = PetscStrchr(loc2_fmt,':',&loc3_fmode);CHKERRQ(ierr);
}
if (loc3_fmode) *loc3_fmode++ = 0;
ierr = PetscStrendswithwhich(*loc0_vtype ? loc0_vtype : "ascii",viewers,&cnt);CHKERRQ(ierr);
if (cnt > (PetscInt) sizeof(viewers)-1) SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Unknown viewer type: %s",loc0_vtype);
if (viewer) {
if (!loc1_fname) {
switch (cnt) {
case 0:
ierr = PetscViewerASCIIGetStdout(comm,viewer);CHKERRQ(ierr);
break;
case 1:
if (!(*viewer = PETSC_VIEWER_BINARY_(comm))) CHKERRQ(PETSC_ERR_PLIB);
break;
case 2:
if (!(*viewer = PETSC_VIEWER_DRAW_(comm))) CHKERRQ(PETSC_ERR_PLIB);
break;
#if defined(PETSC_USE_SOCKET_VIEWER)
case 3:
if (!(*viewer = PETSC_VIEWER_SOCKET_(comm))) CHKERRQ(PETSC_ERR_PLIB);
break;
#endif
#if defined(PETSC_HAVE_MATLAB_ENGINE)
case 4:
if (!(*viewer = PETSC_VIEWER_MATLAB_(comm))) CHKERRQ(PETSC_ERR_PLIB);
break;
#endif
#if defined(PETSC_HAVE_SAWS)
case 5:
if (!(*viewer = PETSC_VIEWER_SAWS_(comm))) CHKERRQ(PETSC_ERR_PLIB);
break;
#endif
#if defined(PETSC_HAVE_HDF5)
case 7:
if (!(*viewer = PETSC_VIEWER_HDF5_(comm))) CHKERRQ(PETSC_ERR_PLIB);
break;
#endif
default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Unsupported viewer %s",loc0_vtype);
}
ierr = PetscObjectReference((PetscObject)*viewer);CHKERRQ(ierr);
} else {
if (loc2_fmt && !*loc1_fname && (cnt == 0)) { /* ASCII format without file name */
ierr = PetscViewerASCIIGetStdout(comm,viewer);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)*viewer);CHKERRQ(ierr);
} else {
PetscFileMode fmode;
ierr = PetscViewerCreate(comm,viewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(*viewer,*loc0_vtype ? loc0_vtype : "ascii");CHKERRQ(ierr);
fmode = FILE_MODE_WRITE;
if (loc3_fmode && *loc3_fmode) { /* Has non-empty file mode ("write" or "append") */
ierr = PetscEnumFind(PetscFileModes,loc3_fmode,(PetscEnum*)&fmode,&flag);CHKERRQ(ierr);
if (!flag) SETERRQ1(comm,PETSC_ERR_ARG_UNKNOWN_TYPE,"Unknown file mode: %s",loc3_fmode);
}
ierr = PetscViewerFileSetMode(*viewer,flag?fmode:FILE_MODE_WRITE);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(*viewer,loc1_fname);CHKERRQ(ierr);
ierr = PetscViewerDrawSetDrawType(*viewer,loc1_fname);CHKERRQ(ierr);
}
}
}
if (viewer) {
ierr = PetscViewerSetUp(*viewer);CHKERRQ(ierr);
}
if (loc2_fmt && *loc2_fmt) {
ierr = PetscEnumFind(PetscViewerFormats,loc2_fmt,(PetscEnum*)format,&flag);CHKERRQ(ierr);
if (!flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Unknown viewer format %s",loc2_fmt);CHKERRQ(ierr);
}
ierr = PetscFree(loc0_vtype);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode KSPComputeShifts_DGMRES(KSP ksp)
{
PetscErrorCode ierr;
KSP_AGMRES *agmres = (KSP_AGMRES*)(ksp->data);
PetscInt max_k = agmres->max_k; /* size of the (non augmented) Krylov subspace */
PetscInt Neig = 0;
PetscInt max_it = ksp->max_it;
/* Perform one cycle of dgmres to find the eigenvalues and compute the first approximations of the eigenvectors */
PetscFunctionBegin;
ierr = PetscLogEventBegin(KSP_AGMRESComputeShifts, ksp, 0,0,0);CHKERRQ(ierr);
/* Send the size of the augmented basis to DGMRES */
ksp->max_it = max_k; /* set this to have DGMRES performing only one cycle */
ksp->ops->buildsolution = KSPBuildSolution_DGMRES;
ierr = KSPSolve_DGMRES(ksp);
ksp->guess_zero = PETSC_FALSE;
if (ksp->reason == KSP_CONVERGED_RTOL) {
ierr = PetscLogEventEnd(KSP_AGMRESComputeShifts, ksp, 0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
} else ksp->reason = KSP_CONVERGED_ITERATING;
if ((agmres->r == 0) && (agmres->neig > 0)) { /* Compute the eigenvalues for the shifts and the eigenvectors (to augment the Newton basis) */
agmres->HasSchur = PETSC_FALSE;
ierr = KSPDGMRESComputeDeflationData_DGMRES (ksp, &Neig);CHKERRQ (ierr);
Neig = max_k;
} else { /* From DGMRES, compute only the eigenvalues needed as Shifts for the Newton Basis */
ierr = KSPDGMRESComputeSchurForm_DGMRES(ksp, &Neig);CHKERRQ(ierr);
}
/* It may happen that the Ritz values from one cycle of GMRES are not accurate enough to provide a good stability. In this case, another cycle of GMRES is performed. The two sets of values thus generated are sorted and the most accurate are kept as shifts */
PetscBool flg;
ierr = PetscOptionsHasName(NULL, "-ksp_agmres_ImproveShifts", &flg);CHKERRQ(ierr);
if (!flg) {
ierr = KSPAGMRESLejaOrdering(agmres->wr, agmres->wi, agmres->Rshift, agmres->Ishift, max_k);CHKERRQ(ierr);
} else { /* Perform another cycle of DGMRES to find another set of eigenvalues */
PetscInt i;
PetscScalar *wr, *wi,*Rshift, *Ishift;
ierr = PetscMalloc4(2*max_k, &wr, 2*max_k, &wi, 2*max_k, &Rshift, 2*max_k, &Ishift);CHKERRQ(ierr);
for (i = 0; i < max_k; i++) {
wr[i] = agmres->wr[i];
wi[i] = agmres->wi[i];
}
ierr = KSPSolve_DGMRES(ksp);
ksp->guess_zero = PETSC_FALSE;
if (ksp->reason == KSP_CONVERGED_RTOL) PetscFunctionReturn(0);
else ksp->reason = KSP_CONVERGED_ITERATING;
if (agmres->neig > 0) { /* Compute the eigenvalues for the shifts) and the eigenvectors (to augment the Newton basis */
agmres->HasSchur = PETSC_FALSE;
ierr = KSPDGMRESComputeDeflationData_DGMRES(ksp, &Neig);CHKERRQ(ierr);
Neig = max_k;
} else { /* From DGMRES, compute only the eigenvalues needed as Shifts for the Newton Basis */
ierr = KSPDGMRESComputeSchurForm_DGMRES(ksp, &Neig);CHKERRQ(ierr);
}
for (i = 0; i < max_k; i++) {
wr[max_k+i] = agmres->wr[i];
wi[max_k+i] = agmres->wi[i];
}
ierr = KSPAGMRESLejaOrdering(wr, wi, Rshift, Ishift, 2*max_k);CHKERRQ(ierr);
for (i = 0; i< max_k; i++) {
agmres->Rshift[i] = Rshift[i];
agmres->Ishift[i] = Ishift[i];
}
ierr = PetscFree(Rshift);CHKERRQ(ierr);
ierr = PetscFree(wr);CHKERRQ(ierr);
ierr = PetscFree(Ishift);CHKERRQ(ierr);
ierr = PetscFree(wi);CHKERRQ(ierr);
}
agmres->HasShifts = PETSC_TRUE;
ksp->max_it = max_it;
ierr = PetscLogEventEnd(KSP_AGMRESComputeShifts, ksp, 0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
Mat A,B,C,D;
PetscInt i,M=10,N=5,j,nrows,ncols,am,an,rstart,rend;
PetscErrorCode ierr;
PetscRandom r;
PetscBool equal,iselemental;
PetscReal fill = 1.0;
IS isrows,iscols;
const PetscInt *rows,*cols;
PetscScalar *v,rval;
#if defined(PETSC_HAVE_ELEMENTAL)
PetscBool Test_MatMatMult=PETSC_TRUE;
#else
PetscBool Test_MatMatMult=PETSC_FALSE;
#endif
PetscMPIInt size;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,M,N);CHKERRQ(ierr);
ierr = MatSetType(A,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&r);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(r);CHKERRQ(ierr);
/* Set local matrix entries */
ierr = MatGetOwnershipIS(A,&isrows,&iscols);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
ierr = PetscRandomGetValue(r,&rval);CHKERRQ(ierr);
v[i*ncols+j] = rval;
}
}
ierr = MatSetValues(A,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&r);CHKERRQ(ierr);
/* Test MatMatMult() */
if (Test_MatMatMult) {
#if !defined(PETSC_HAVE_ELEMENTAL)
if (size > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"This test requires ELEMENTAL");
#endif
ierr = MatTranspose(A,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr); /* B = A^T */
ierr = MatMatMult(B,A,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr); /* C = B*A = A^T*A */
ierr = MatMatMult(B,A,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
/* Test B*A*x = C*x for n random vector x */
ierr = MatMatMultEqual(B,A,C,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"B*A*x != C*x");
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatMatMultSymbolic(B,A,fill,&C);CHKERRQ(ierr);
for (i=0; i<2; i++) {
/* Repeat the numeric product to test reuse of the previous symbolic product */
ierr = MatMatMultNumeric(B,A,C);CHKERRQ(ierr);
ierr = MatMatMultEqual(B,A,C,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"B*A*x != C*x");
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
}
/* Test MatTransposeMatMult() */
ierr = PetscObjectTypeCompare((PetscObject)A,MATELEMENTAL,&iselemental);CHKERRQ(ierr);
if (!iselemental) {
ierr = MatTransposeMatMult(A,A,MAT_INITIAL_MATRIX,fill,&D);CHKERRQ(ierr); /* D = A^T*A */
ierr = MatTransposeMatMult(A,A,MAT_REUSE_MATRIX,fill,&D);CHKERRQ(ierr);
/* ierr = MatView(D,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = MatTransposeMatMultEqual(A,A,D,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"D*x != A^T*A*x");
ierr = MatDestroy(&D);CHKERRQ(ierr);
/* Test D*x = A^T*C*A*x, where C is in AIJ format */
ierr = MatGetLocalSize(A,&am,&an);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
if (size == 1) {
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,am,am);CHKERRQ(ierr);
} else {
ierr = MatSetSizes(C,am,am,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
}
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(C,&rstart,&rend);CHKERRQ(ierr);
v[0] = 1.0;
for (i=rstart; i<rend; i++) {
ierr = MatSetValues(C,1,&i,1,&i,v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* B = C*A, D = A^T*B */
ierr = MatMatMult(C,A,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
ierr = MatTransposeMatMult(A,B,MAT_INITIAL_MATRIX,fill,&D);CHKERRQ(ierr);
ierr = MatTransposeMatMultEqual(A,B,D,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"D*x != A^T*B*x");
ierr = MatDestroy(&D);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
PetscFinalize();
return(0);
}
/*@
PetscBinaryRead - Reads from a binary file.
Not Collective
Input Parameters:
+ fd - the file
. n - the number of items to read
- type - the type of items to read (PETSC_INT, PETSC_DOUBLE or PETSC_SCALAR)
Output Parameters:
. p - the buffer
Level: developer
Notes:
PetscBinaryRead() uses byte swapping to work on all machines; the files
are written to file ALWAYS using big-endian ordering. On small-endian machines the numbers
are converted to the small-endian format when they are read in from the file.
When PETSc is ./configure with --with-64bit-indices the integers are written to the
file as 64 bit integers, this means they can only be read back in when the option --with-64bit-indices
is used.
Concepts: files^reading binary
Concepts: binary files^reading
.seealso: PetscBinaryWrite(), PetscBinaryOpen(), PetscBinaryClose(), PetscViewerBinaryGetDescriptor(), PetscBinarySynchronizedWrite(),
PetscBinarySynchronizedRead(), PetscBinarySynchronizedSeek()
@*/
PetscErrorCode PetscBinaryRead(int fd,void *p,PetscInt n,PetscDataType type)
{
int wsize,err;
size_t m = (size_t) n,maxblock = 65536;
char *pp = (char*)p;
#if defined(PETSC_USE_REAL___FLOAT128)
PetscBool readdouble = PETSC_FALSE;
double *ppp;
#endif
#if !defined(PETSC_WORDS_BIGENDIAN) || defined(PETSC_USE_REAL___FLOAT128)
PetscErrorCode ierr;
#endif
#if !defined(PETSC_WORDS_BIGENDIAN)
void *ptmp = p;
#endif
char *fname = NULL;
PetscFunctionBegin;
if (n < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Trying to write a negative amount of data %D",n);
if (!n) PetscFunctionReturn(0);
if (type == PETSC_FUNCTION) {
m = 64;
type = PETSC_CHAR;
fname = (char*) malloc(m*sizeof(char));
if (!fname) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Cannot allocate space for function name");
pp = (char*)fname;
#if !defined(PETSC_WORDS_BIGENDIAN)
ptmp = (void*)fname;
#endif
}
if (type == PETSC_INT) m *= sizeof(PetscInt);
else if (type == PETSC_SCALAR) m *= sizeof(PetscScalar);
else if (type == PETSC_DOUBLE) m *= sizeof(double);
else if (type == PETSC_FLOAT) m *= sizeof(float);
else if (type == PETSC_SHORT) m *= sizeof(short);
else if (type == PETSC_CHAR) m *= sizeof(char);
else if (type == PETSC_ENUM) m *= sizeof(PetscEnum);
else if (type == PETSC_BOOL) m *= sizeof(PetscBool);
else if (type == PETSC_BIT_LOGICAL) m = PetscBTLength(m)*sizeof(char);
else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Unknown type");
#if defined(PETSC_USE_REAL___FLOAT128)
ierr = PetscOptionsGetBool(NULL,"-binary_read_double",&readdouble,NULL);CHKERRQ(ierr);
/* If using __float128 precision we still read in doubles from file */
if (type == PETSC_SCALAR && readdouble) {
m = m/2;
ierr = PetscMalloc1(n,&ppp);CHKERRQ(ierr);
pp = (char*)ppp;
}
#endif
while (m) {
wsize = (m < maxblock) ? m : maxblock;
err = read(fd,pp,wsize);
if (err < 0 && errno == EINTR) continue;
if (!err && wsize > 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_FILE_READ,"Read past end of file");
if (err < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_READ,"Error reading from file, errno %d",errno);
m -= err;
pp += err;
}
#if defined(PETSC_USE_REAL___FLOAT128)
if (type == PETSC_SCALAR && readdouble) {
PetscScalar *pv = (PetscScalar*) p;
PetscInt i;
#if !defined(PETSC_WORDS_BIGENDIAN)
ierr = PetscByteSwapDouble(ppp,n);CHKERRQ(ierr);
#endif
for (i=0; i<n; i++) pv[i] = ppp[i];
ierr = PetscFree(ppp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
#if !defined(PETSC_WORDS_BIGENDIAN)
ierr = PetscByteSwap(ptmp,type,n);CHKERRQ(ierr);
#endif
if (type == PETSC_FUNCTION) {
#if defined(PETSC_SERIALIZE_FUNCTIONS)
ierr = PetscDLSym(NULL,fname,(void**)p);CHKERRQ(ierr);
#else
*(void**)p = NULL;
#endif
free(fname);
}
PetscFunctionReturn(0);
}
PetscErrorCode FormMatrix(DM da,Mat jac)
{
PetscErrorCode ierr;
PetscInt i,j,nrows = 0;
MatStencil col[5],row,*rows;
PetscScalar v[5],hx,hy,hxdhy,hydhx;
DMDALocalInfo info;
PetscFunctionBegin;
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
hx = 1.0/(PetscReal)(info.mx-1);
hy = 1.0/(PetscReal)(info.my-1);
hxdhy = hx/hy;
hydhx = hy/hx;
ierr = PetscMalloc(info.ym*info.xm*sizeof(MatStencil),&rows);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Here, we set all entries for a particular row at once.
- We can set matrix entries either using either
MatSetValuesLocal() or MatSetValues(), as discussed above.
*/
for (j=info.ys; j<info.ys+info.ym; j++) {
for (i=info.xs; i<info.xs+info.xm; i++) {
row.j = j; row.i = i;
/* boundary points */
if (i == 0 || j == 0 || i == info.mx-1 || j == info.my-1) {
v[0] = 2.0*(hydhx + hxdhy);
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
rows[nrows].i = i;
rows[nrows++].j = j;
} else {
/* interior grid points */
v[0] = -hxdhy; col[0].j = j - 1; col[0].i = i;
v[1] = -hydhx; col[1].j = j; col[1].i = i-1;
v[2] = 2.0*(hydhx + hxdhy); col[2].j = row.j; col[2].i = row.i;
v[3] = -hydhx; col[3].j = j; col[3].i = i+1;
v[4] = -hxdhy; col[4].j = j + 1; col[4].i = i;
ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsStencil(jac,nrows,rows,2.0*(hydhx + hxdhy),PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
/*
Tell the matrix we will never add a new nonzero location to the
matrix. If we do, it will generate an error.
*/
ierr = MatSetOption(jac,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
* The interface should be easy to use for both MatGetSubMatrix (parallel sub-matrix) and MatGetSubMatrices (sequential sub-matrices)
* */
static PetscErrorCode MatGetSubMatrix_MPIAdj_data(Mat adj,IS irows, IS icols, PetscInt **sadj_xadj,PetscInt **sadj_adjncy,PetscInt **sadj_values)
{
PetscInt nlrows_is,icols_n,i,j,nroots,nleaves,owner,rlocalindex,*ncols_send,*ncols_recv;
PetscInt nlrows_mat,*adjncy_recv,Ncols_recv,Ncols_send,*xadj_recv,*values_recv;
PetscInt *ncols_recv_offsets,loc,rnclos,*sadjncy,*sxadj,*svalues,isvalue;
const PetscInt *irows_indices,*icols_indices,*xadj, *adjncy;
Mat_MPIAdj *a = (Mat_MPIAdj*)adj->data;
PetscLayout rmap;
MPI_Comm comm;
PetscSF sf;
PetscSFNode *iremote;
PetscBool done;
PetscErrorCode ierr;
PetscFunctionBegin;
/* communicator */
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
/* Layouts */
ierr = MatGetLayouts(adj,&rmap,PETSC_NULL);CHKERRQ(ierr);
/* get rows information */
ierr = ISGetLocalSize(irows,&nlrows_is);CHKERRQ(ierr);
ierr = ISGetIndices(irows,&irows_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(nlrows_is,&iremote);CHKERRQ(ierr);
/* construct sf graph*/
nleaves = nlrows_is;
for(i=0; i<nlrows_is; i++){
owner = -1;
rlocalindex = -1;
ierr = PetscLayoutFindOwnerIndex(rmap,irows_indices[i],&owner,&rlocalindex);CHKERRQ(ierr);
iremote[i].rank = owner;
iremote[i].index = rlocalindex;
}
ierr = MatGetRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);CHKERRQ(ierr);
ierr = PetscCalloc4(nlrows_mat,&ncols_send,nlrows_is,&xadj_recv,nlrows_is+1,&ncols_recv_offsets,nlrows_is,&ncols_recv);CHKERRQ(ierr);
nroots = nlrows_mat;
for(i=0; i<nlrows_mat; i++){
ncols_send[i] = xadj[i+1]-xadj[i];
}
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,ncols_send,ncols_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,ncols_send,ncols_recv);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,xadj,xadj_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,xadj,xadj_recv);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
Ncols_recv =0;
for(i=0; i<nlrows_is; i++){
Ncols_recv += ncols_recv[i];
ncols_recv_offsets[i+1] = ncols_recv[i]+ncols_recv_offsets[i];
}
Ncols_send = 0;
for(i=0; i<nlrows_mat; i++){
Ncols_send += ncols_send[i];
}
ierr = PetscCalloc1(Ncols_recv,&iremote);CHKERRQ(ierr);
ierr = PetscCalloc1(Ncols_recv,&adjncy_recv);CHKERRQ(ierr);
nleaves = Ncols_recv;
Ncols_recv = 0;
for(i=0; i<nlrows_is; i++){
ierr = PetscLayoutFindOwner(rmap,irows_indices[i],&owner);CHKERRQ(ierr);
for(j=0; j<ncols_recv[i]; j++){
iremote[Ncols_recv].rank = owner;
iremote[Ncols_recv++].index = xadj_recv[i]+j;
}
}
ierr = ISRestoreIndices(irows,&irows_indices);CHKERRQ(ierr);
/*if we need to deal with edge weights ???*/
if(a->values){isvalue=1;}else{isvalue=0;}
/*involve a global communication */
/*ierr = MPI_Allreduce(&isvalue,&isvalue,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr);*/
if(isvalue){ierr = PetscCalloc1(Ncols_recv,&values_recv);CHKERRQ(ierr);}
nroots = Ncols_send;
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,adjncy,adjncy_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,adjncy,adjncy_recv);CHKERRQ(ierr);
if(isvalue){
ierr = PetscSFBcastBegin(sf,MPIU_INT,a->values,values_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,a->values,values_recv);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = MatRestoreRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);CHKERRQ(ierr);
ierr = ISGetLocalSize(icols,&icols_n);CHKERRQ(ierr);
ierr = ISGetIndices(icols,&icols_indices);CHKERRQ(ierr);
rnclos = 0;
for(i=0; i<nlrows_is; i++){
for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
ierr = PetscFindInt(adjncy_recv[j], icols_n, icols_indices, &loc);CHKERRQ(ierr);
if(loc<0){
adjncy_recv[j] = -1;
if(isvalue) values_recv[j] = -1;
ncols_recv[i]--;
}else{
rnclos++;
}
}
}
ierr = ISRestoreIndices(icols,&icols_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(rnclos,&sadjncy);CHKERRQ(ierr);
if(isvalue) {ierr = PetscCalloc1(rnclos,&svalues);CHKERRQ(ierr);}
ierr = PetscCalloc1(nlrows_is+1,&sxadj);CHKERRQ(ierr);
rnclos = 0;
for(i=0; i<nlrows_is; i++){
for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
if(adjncy_recv[j]<0) continue;
sadjncy[rnclos] = adjncy_recv[j];
if(isvalue) svalues[rnclos] = values_recv[j];
rnclos++;
}
}
for(i=0; i<nlrows_is; i++){
sxadj[i+1] = sxadj[i]+ncols_recv[i];
}
if(sadj_xadj) { *sadj_xadj = sxadj;}else { ierr = PetscFree(sxadj);CHKERRQ(ierr);}
if(sadj_adjncy){ *sadj_adjncy = sadjncy;}else{ ierr = PetscFree(sadjncy);CHKERRQ(ierr);}
if(sadj_values){
if(isvalue) *sadj_values = svalues; else *sadj_values=0;
}else{
if(isvalue) {ierr = PetscFree(svalues);CHKERRQ(ierr);}
}
ierr = PetscFree4(ncols_send,xadj_recv,ncols_recv_offsets,ncols_recv);CHKERRQ(ierr);
ierr = PetscFree(adjncy_recv);CHKERRQ(ierr);
if(isvalue) {ierr = PetscFree(values_recv);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
PetscErrorCode ComputeMatrix(DM da,Mat B)
{
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs,dof,k1,k2,k3;
PetscScalar *v,*v_neighbor,Hx,Hy,Hz,HxHydHz,HyHzdHx,HxHzdHy,r1,r2;
MatStencil row,col;
PetscRandom rand;
PetscFunctionBegin;
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetType(rand,PETSCRAND);CHKERRQ(ierr);
ierr = PetscRandomSetSeed(rand,1);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(rand,-.001,.001);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,&my,&mz,0,0,0,&dof,0,0,0,0,0);CHKERRQ(ierr);
/* For simplicity, this example only works on mx=my=mz */
if (mx != my || mx != mz) SETERRQ3(PETSC_COMM_SELF,1,"This example only works with mx %d = my %d = mz %d\n",mx,my,mz);
Hx = 1.0 / (PetscReal)(mx-1); Hy = 1.0 / (PetscReal)(my-1); Hz = 1.0 / (PetscReal)(mz-1);
HxHydHz = Hx*Hy/Hz; HxHzdHy = Hx*Hz/Hy; HyHzdHx = Hy*Hz/Hx;
ierr = PetscMalloc1((2*dof*dof+1),&v);CHKERRQ(ierr);
v_neighbor = v + dof*dof;
ierr = PetscMemzero(v,(2*dof*dof+1)*sizeof(PetscScalar));CHKERRQ(ierr);
k3 = 0;
for (k1=0; k1<dof; k1++) {
for (k2=0; k2<dof; k2++) {
if (k1 == k2) {
v[k3] = 2.0*(HxHydHz + HxHzdHy + HyHzdHx);
v_neighbor[k3] = -HxHydHz;
} else {
ierr = PetscRandomGetValue(rand,&r1);CHKERRQ(ierr);
ierr = PetscRandomGetValue(rand,&r2);CHKERRQ(ierr);
v[k3] = r1;
v_neighbor[k3] = r2;
}
k3++;
}
}
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j; row.k = k;
if (i==0 || j==0 || k==0 || i==mx-1 || j==my-1 || k==mz-1) { /* boudary points */
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else { /* interior points */
/* center */
col.i = i; col.j = j; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v,INSERT_VALUES);CHKERRQ(ierr);
/* x neighbors */
col.i = i-1; col.j = j; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
col.i = i+1; col.j = j; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
/* y neighbors */
col.i = i; col.j = j-1; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
col.i = i; col.j = j+1; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
/* z neighbors */
col.i = i; col.j = j; col.k = k-1;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
col.i = i; col.j = j; col.k = k+1;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
}
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
MatNullSpaceCreate - Creates a data structure used to project vectors
out of null spaces.
Collective on MPI_Comm
Input Parameters:
+ comm - the MPI communicator associated with the object
. has_cnst - PETSC_TRUE if the null space contains the constant vector; otherwise PETSC_FALSE
. n - number of vectors (excluding constant vector) in null space
- vecs - the vectors that span the null space (excluding the constant vector);
these vectors must be orthonormal. These vectors are NOT copied, so do not change them
after this call. You should free the array that you pass in and destroy the vectors (this will reduce the reference count
for them by one).
Output Parameter:
. SP - the null space context
Level: advanced
Notes: See MatNullSpaceSetFunction() as an alternative way of providing the null space information instead of setting vecs.
If has_cnst is PETSC_TRUE you do not need to pass a constant vector in as a fourth argument to this routine, nor do you
need to pass in a function that eliminates the constant function into MatNullSpaceSetFunction().
Users manual sections:
. sec_singular
.keywords: PC, null space, create
.seealso: MatNullSpaceDestroy(), MatNullSpaceRemove(), MatSetNullSpace(), MatNullSpace, MatNullSpaceSetFunction()
@*/
PetscErrorCode MatNullSpaceCreate(MPI_Comm comm,PetscBool has_cnst,PetscInt n,const Vec vecs[],MatNullSpace *SP)
{
MatNullSpace sp;
PetscErrorCode ierr;
PetscInt i;
PetscFunctionBegin;
if (n < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Number of vectors (given %D) cannot be negative",n);
if (n) PetscValidPointer(vecs,4);
for (i=0; i<n; i++) PetscValidHeaderSpecific(vecs[i],VEC_CLASSID,4);
PetscValidPointer(SP,5);
if (n) {
for (i=0; i<n; i++) {
/* prevent the user from changes values in the vector */
ierr = VecLockPush(vecs[i]);CHKERRQ(ierr);
}
}
#if defined(PETSC_USE_DEBUG)
if (n) {
PetscScalar *dots;
for (i=0; i<n; i++) {
PetscReal norm;
ierr = VecNorm(vecs[i],NORM_2,&norm);CHKERRQ(ierr);
if (PetscAbsReal(norm - 1.0) > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PetscObjectComm((PetscObject)vecs[i]),PETSC_ERR_ARG_WRONG,"Vector %D must have 2-norm of 1.0, it is %g",i,(double)norm);
}
if (has_cnst) {
for (i=0; i<n; i++) {
PetscScalar sum;
ierr = VecSum(vecs[i],&sum);CHKERRQ(ierr);
if (PetscAbsScalar(sum) > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PetscObjectComm((PetscObject)vecs[i]),PETSC_ERR_ARG_WRONG,"Vector %D must be orthogonal to constant vector, inner product is %g",i,(double)PetscAbsScalar(sum));
}
}
ierr = PetscMalloc1(n-1,&dots);CHKERRQ(ierr);
for (i=0; i<n-1; i++) {
PetscInt j;
ierr = VecMDot(vecs[i],n-i-1,vecs+i+1,dots);CHKERRQ(ierr);
for (j=0;j<n-i-1;j++) {
if (PetscAbsScalar(dots[j]) > PETSC_SQRT_MACHINE_EPSILON) SETERRQ3(PetscObjectComm((PetscObject)vecs[i]),PETSC_ERR_ARG_WRONG,"Vector %D must be orthogonal to vector %D, inner product is %g",i,i+j+1,(double)PetscAbsScalar(dots[j]));
}
}
PetscFree(dots);CHKERRQ(ierr);
}
#endif
*SP = NULL;
ierr = MatInitializePackage();CHKERRQ(ierr);
ierr = PetscHeaderCreate(sp,MAT_NULLSPACE_CLASSID,"MatNullSpace","Null space","Mat",comm,MatNullSpaceDestroy,MatNullSpaceView);CHKERRQ(ierr);
sp->has_cnst = has_cnst;
sp->n = n;
sp->vecs = 0;
sp->alpha = 0;
sp->remove = 0;
sp->rmctx = 0;
if (n) {
ierr = PetscMalloc1(n,&sp->vecs);CHKERRQ(ierr);
ierr = PetscMalloc1(n,&sp->alpha);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)sp,n*(sizeof(Vec)+sizeof(PetscScalar)));CHKERRQ(ierr);
for (i=0; i<n; i++) {
ierr = PetscObjectReference((PetscObject)vecs[i]);CHKERRQ(ierr);
sp->vecs[i] = vecs[i];
}
}
*SP = sp;
PetscFunctionReturn(0);
}
PetscErrorCode FACreate(FA *infa)
{
FA fa;
PetscMPIInt rank;
PetscInt tonglobal,globalrstart,x,nx,y,ny,*tonatural,i,j,*to,*from,offt[3];
PetscInt *fromnatural,fromnglobal,nscat,nlocal,cntl1,cntl2,cntl3,*indices;
PetscErrorCode ierr;
/* Each DMDA manages the local vector for the portion of region 1, 2, and 3 for that processor
Each DMDA can belong on any subset (overlapping between DMDA's or not) of processors
For processes that a particular DMDA does not exist on, the corresponding comm should be set to zero
*/
DM da1 = 0,da2 = 0,da3 = 0;
/*
v1, v2, v3 represent the local vector for a single DMDA
*/
Vec vl1 = 0,vl2 = 0,vl3 = 0, vg1 = 0, vg2 = 0,vg3 = 0;
/*
globalvec and friends represent the global vectors that are used for the PETSc solvers
localvec represents the concatenation of the (up to) 3 local vectors; vl1, vl2, vl3
tovec and friends represent intermediate vectors that are ONLY used for setting up the
final communication patterns. Once this setup routine is complete they are destroyed.
The tovec is like the globalvec EXCEPT it has redundant locations for the ghost points
between regions 2+3 and 1.
*/
AO toao,globalao;
IS tois,globalis,is;
Vec tovec,globalvec,localvec;
VecScatter vscat;
PetscScalar *globalarray,*localarray,*toarray;
ierr = PetscNew(struct _p_FA,&fa);CHKERRQ(ierr);
/*
fa->sw is the stencil width
fa->p1 is the width of region 1, fa->p2 the width of region 2 (must be the same)
fa->r1 height of region 1
fa->r2 height of region 2
fa->r2 is also the height of region 3-4
(fa->p1 - fa->p2)/2 is the width of both region 3 and region 4
*/
fa->p1 = 24;
fa->p2 = 15;
fa->r1 = 6;
fa->r2 = 6;
fa->sw = 1;
fa->r1g = fa->r1 + fa->sw;
fa->r2g = fa->r2 + fa->sw;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
fa->comm[0] = PETSC_COMM_WORLD;
fa->comm[1] = PETSC_COMM_WORLD;
fa->comm[2] = PETSC_COMM_WORLD;
/* Test case with different communicators */
/* Normally one would use MPI_Comm routines to build MPI communicators on which you wish to partition the DMDAs*/
/*
if (!rank) {
fa->comm[0] = PETSC_COMM_SELF;
fa->comm[1] = 0;
fa->comm[2] = 0;
} else if (rank == 1) {
fa->comm[0] = 0;
fa->comm[1] = PETSC_COMM_SELF;
fa->comm[2] = 0;
} else {
fa->comm[0] = 0;
fa->comm[1] = 0;
fa->comm[2] = PETSC_COMM_SELF;
} */
if (fa->p2 > fa->p1 - 3) SETERRQ(PETSC_COMM_SELF,1,"Width of region fa->p2 must be at least 3 less then width of region 1");
if (!((fa->p2 - fa->p1) % 2)) SETERRQ(PETSC_COMM_SELF,1,"width of region 3 must NOT be divisible by 2");
if (fa->comm[1]) {
ierr = DMDACreate2d(fa->comm[1],DMDA_BOUNDARY_PERIODIC,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da2);CHKERRQ(ierr);
ierr = DMGetLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMDACreate2d(fa->comm[2],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1-fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da3);CHKERRQ(ierr);
ierr = DMGetLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = DMDACreate2d(fa->comm[0],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1,fa->r1g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da1);CHKERRQ(ierr);
ierr = DMGetLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector with redundancy */
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,tonglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get tonatural number for each node */
ierr = PetscMalloc((tonglobal+1)*sizeof(PetscInt),&tonatural);CHKERRQ(ierr);
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) tonatural[tonglobal++] = x + i + fa->p1*(y + j);
else tonatural[tonglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
}
}
}
/* ierr = PetscIntView(tonglobal,tonatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = AOCreateBasic(PETSC_COMM_WORLD,tonglobal,tonatural,0,&toao);CHKERRQ(ierr);
ierr = PetscFree(tonatural);CHKERRQ(ierr);
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector without redundancy */
fromnglobal = 0;
fa->offg[1] = 0;
offt[1] = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[2] = nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[2] = fromnglobal;
} else {
offt[2] = 0;
fa->offg[2] = 0;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[0] = offt[2] + nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[0] = fromnglobal;
} else {
offt[0] = offt[2];
fa->offg[0] = fromnglobal;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
fromnglobal += nx*ny;
}
ierr = MPI_Scan(&fromnglobal,&globalrstart,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
globalrstart -= fromnglobal;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,fromnglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get fromnatural number for each node */
ierr = PetscMalloc((fromnglobal+1)*sizeof(PetscInt),&fromnatural);CHKERRQ(ierr);
fromnglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[1] = x; fa->yg[1] = y; fa->mg[1] = nx; fa->ng[1] = ny;
ierr = DMDAGetGhostCorners(da2,&fa->xl[1],&fa->yl[1],0,&fa->ml[1],&fa->nl[1],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[2] = x; fa->yg[2] = y; fa->mg[2] = nx; fa->ng[2] = ny;
ierr = DMDAGetGhostCorners(da3,&fa->xl[2],&fa->yl[2],0,&fa->ml[2],&fa->nl[2],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) fromnatural[fromnglobal++] = x + i + fa->p1*(y + j);
else fromnatural[fromnglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
else y--;
fa->xg[0] = x; fa->yg[0] = y; fa->mg[0] = nx; fa->ng[0] = ny;
ierr = DMDAGetGhostCorners(da1,&fa->xl[0],&fa->yl[0],0,&fa->ml[0],&fa->nl[0],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = fa->p1*fa->r2 + x + i + fa->p1*(y + j);
}
}
}
/*ierr = PetscIntView(fromnglobal,fromnatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
ierr = AOCreateBasic(PETSC_COMM_WORLD,fromnglobal,fromnatural,0,&globalao);CHKERRQ(ierr);
ierr = PetscFree(fromnatural);CHKERRQ(ierr);
/* ---------------------------------------------------*/
/* Create the scatter that updates 1 from 2 and 3 and 3 and 2 from 1 */
/* currently handles stencil width of 1 ONLY */
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&to);CHKERRQ(ierr);
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&from);CHKERRQ(ierr);
nscat = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = from[nscat] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) {
to[nscat] = from[nscat] = x + i + fa->p1*(y + j);nscat++;
} else {
to[nscat] = from[nscat] = fa->p2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
from[nscat++] = fa->p1*(fa->r2 - 1) + x + i + fa->p1*(y + j);
}
}
}
ierr = AOApplicationToPetsc(toao,nscat,to);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(globalao,nscat,from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,to,PETSC_COPY_VALUES,&tois);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,from,PETSC_COPY_VALUES,&globalis);CHKERRQ(ierr);
ierr = PetscFree(to);CHKERRQ(ierr);
ierr = PetscFree(from);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,tonglobal,PETSC_DETERMINE,&tovec);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,fromnglobal,PETSC_DETERMINE,&globalvec);CHKERRQ(ierr);
ierr = VecScatterCreate(globalvec,globalis,tovec,tois,&vscat);CHKERRQ(ierr);
ierr = ISDestroy(&tois);CHKERRQ(ierr);
ierr = ISDestroy(&globalis);CHKERRQ(ierr);
ierr = AODestroy(&globalao);CHKERRQ(ierr);
ierr = AODestroy(&toao);CHKERRQ(ierr);
/* fill up global vector without redundant values with PETSc global numbering */
ierr = VecGetArray(globalvec,&globalarray);CHKERRQ(ierr);
for (i=0; i<fromnglobal; i++) {
globalarray[i] = globalrstart + i;
}
ierr = VecRestoreArray(globalvec,&globalarray);CHKERRQ(ierr);
/* scatter PETSc global indices to redundant valueed array */
ierr = VecScatterBegin(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Create local vector that is the concatenation of the local vectors */
nlocal = 0;
cntl1 = cntl2 = cntl3 = 0;
if (fa->comm[1]) {
ierr = VecGetSize(vl2,&cntl2);CHKERRQ(ierr);
nlocal += cntl2;
}
if (fa->comm[2]) {
ierr = VecGetSize(vl3,&cntl3);CHKERRQ(ierr);
nlocal += cntl3;
}
if (fa->comm[0]) {
ierr = VecGetSize(vl1,&cntl1);CHKERRQ(ierr);
nlocal += cntl1;
}
fa->offl[0] = cntl2 + cntl3;
fa->offl[1] = 0;
fa->offl[2] = cntl2;
ierr = VecCreateSeq(PETSC_COMM_SELF,nlocal,&localvec);CHKERRQ(ierr);
/* cheat so that vl1, vl2, vl3 shared array memory with localvec */
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecGetArray(tovec,&toarray);CHKERRQ(ierr);
if (fa->comm[1]) {
ierr = VecPlaceArray(vl2,localarray+fa->offl[1]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg2,toarray+offt[1]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = VecPlaceArray(vl3,localarray+fa->offl[2]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg3,toarray+offt[2]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = VecPlaceArray(vl1,localarray+fa->offl[0]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg1,toarray+offt[0]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecRestoreArray(tovec,&toarray);CHKERRQ(ierr);
/* no longer need the redundant vector and VecScatter to it */
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
ierr = VecDestroy(&tovec);CHKERRQ(ierr);
/* Create final scatter that goes directly from globalvec to localvec */
/* this is the one to be used in the application code */
ierr = PetscMalloc(nlocal*sizeof(PetscInt),&indices);CHKERRQ(ierr);
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
for (i=0; i<nlocal; i++) {
indices[i] = (PetscInt) (localarray[i]);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_WORLD,2,nlocal,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,2*nlocal,&fa->l);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,2*fromnglobal,PETSC_DETERMINE,&fa->g);CHKERRQ(ierr);
ierr = VecScatterCreate(fa->g,is,fa->l,NULL,&fa->vscat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecDestroy(&globalvec);CHKERRQ(ierr);
ierr = VecDestroy(&localvec);CHKERRQ(ierr);
if (fa->comm[0]) {
ierr = DMRestoreLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMDestroy(&da1);CHKERRQ(ierr);
}
if (fa->comm[1]) {
ierr = DMRestoreLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMRestoreLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMDestroy(&da3);CHKERRQ(ierr);
}
*infa = fa;
PetscFunctionReturn(0);
}
