PetscErrorCode MatMatMultNumeric_MPIAIJ_MPIAIJ(Mat A,Mat B,Mat C)
{
PetscErrorCode ierr;
Mat *seq;
Mat_MatMatMultMPI *mult;
PetscContainer container;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)C,"Mat_MatMatMultMPI",(PetscObject *)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void **)&mult);CHKERRQ(ierr);
} else {
SETERRQ(PETSC_ERR_PLIB,"Container does not exit");
}
seq = &mult->B_seq;
ierr = MatGetSubMatrices(B,1,&mult->isrowb,&mult->iscolb,MAT_REUSE_MATRIX,&seq);CHKERRQ(ierr);
mult->B_seq = *seq;
seq = &mult->A_loc;
ierr = MatGetSubMatrices(A,1,&mult->isrowa,&mult->isrowb,MAT_REUSE_MATRIX,&seq);CHKERRQ(ierr);
mult->A_loc = *seq;
ierr = MatMatMult_SeqAIJ_SeqAIJ(mult->A_loc,mult->B_seq,MAT_REUSE_MATRIX,0.0,&mult->C_seq);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)mult->C_seq);CHKERRQ(ierr);
ierr = MatMerge(((PetscObject)A)->comm,mult->C_seq,B->cmap->n,MAT_REUSE_MATRIX,&C);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,B,*Bsub;
PetscInt i,j,m = 6,n = 6,N = 36,Ii,J;
PetscErrorCode ierr;
PetscScalar v;
IS isrow;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD,N,N,5,NULL,&A);CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=0; j<n; j++) {
v = -1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/* take the first diagonal block */
ierr = ISCreateStride(PETSC_COMM_WORLD,m,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&Bsub);CHKERRQ(ierr);
B = *Bsub;
ierr = PetscFree(Bsub);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatView(B,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
/* take a strided block */
ierr = ISCreateStride(PETSC_COMM_WORLD,m,0,2,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&Bsub);CHKERRQ(ierr);
B = *Bsub;
ierr = PetscFree(Bsub);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatView(B,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
/* take the last block */
ierr = ISCreateStride(PETSC_COMM_WORLD,m,N-m-1,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&Bsub);CHKERRQ(ierr);
B = *Bsub;
ierr = PetscFree(Bsub);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatView(B,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat BAIJ,SBAIJ,*subBAIJ,*subSBAIJ;
PetscViewer viewer;
char file[PETSC_MAX_PATH_LEN];
PetscBool flg;
PetscErrorCode ierr;
PetscInt n = 2,issize,M,N;
PetscMPIInt rank;
IS isrow,iscol,irow[n],icol[n];
PetscInitialize(&argc,&args,(char*)0,help);
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&BAIJ);CHKERRQ(ierr);
ierr = MatSetType(BAIJ,MATMPIBAIJ);CHKERRQ(ierr);
ierr = MatLoad(BAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&SBAIJ);CHKERRQ(ierr);
ierr = MatSetType(SBAIJ,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatLoad(SBAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = MatGetSize(BAIJ,&M,&N);CHKERRQ(ierr);
issize = M/4;
ierr = ISCreateStride(PETSC_COMM_SELF,issize,0,1,&isrow);CHKERRQ(ierr);
irow[0] = irow[1] = isrow;
issize = N/2;
ierr = ISCreateStride(PETSC_COMM_SELF,issize,0,1,&iscol);CHKERRQ(ierr);
icol[0] = icol[1] = iscol;
ierr = MatGetSubMatrices(BAIJ,n,irow,icol,MAT_INITIAL_MATRIX,&subBAIJ);CHKERRQ(ierr);
/* irow and icol must be same for SBAIJ matrices! */
icol[0] = icol[1] = isrow;
ierr = MatGetSubMatrices(SBAIJ,n,irow,icol,MAT_INITIAL_MATRIX,&subSBAIJ);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (!rank) {
ierr = MatView(subBAIJ[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatView(subSBAIJ[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
/* Free data structures */
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subBAIJ);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subSBAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&BAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&SBAIJ);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat BAIJ,SBAIJ,*subBAIJ,*subSBAIJ;
PetscViewer viewer;
char file[PETSC_MAX_PATH_LEN];
PetscBool flg;
PetscErrorCode ierr;
PetscInt n = 2,issize;
PetscMPIInt rank;
IS is,iss[2];
PetscInitialize(&argc,&args,(char*)0,help);
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&BAIJ);CHKERRQ(ierr);
ierr = MatSetType(BAIJ,MATMPIBAIJ);CHKERRQ(ierr);
ierr = MatLoad(BAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&SBAIJ);CHKERRQ(ierr);
ierr = MatSetType(SBAIJ,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatLoad(SBAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = MatGetSize(BAIJ,&issize,0);CHKERRQ(ierr);
issize = 9;
ierr = ISCreateStride(PETSC_COMM_SELF,issize,0,1,&is);CHKERRQ(ierr);
iss[0] = is;iss[1] = is;
ierr = MatGetSubMatrices(BAIJ,n,iss,iss,MAT_INITIAL_MATRIX,&subBAIJ);CHKERRQ(ierr);
ierr = MatGetSubMatrices(SBAIJ,n,iss,iss,MAT_INITIAL_MATRIX,&subSBAIJ);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
#if defined(PETSC_USE_SOCKET_VIEWER)
if (!rank) {
ierr = MatView(subBAIJ[0],PETSC_VIEWER_SOCKET_SELF);CHKERRQ(ierr);
ierr = MatView(subSBAIJ[0],PETSC_VIEWER_SOCKET_SELF);CHKERRQ(ierr);
}
#endif
/* Free data structures */
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subBAIJ);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subSBAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&BAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&SBAIJ);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatPartitioningHierarchical_AssembleSubdomain(Mat adj,IS destination,Mat *sadj, ISLocalToGlobalMapping *mapping)
{
IS irows,icols;
PetscInt irows_ln;
PetscMPIInt rank;
const PetscInt *irows_indices;
MPI_Comm comm;
PetscErrorCode ierr;
PetscFunctionBegin;
/*get comm*/
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
/*get rows from remote and local */
ierr = ISBuildTwoSided(destination,NULL,&irows);CHKERRQ(ierr);
ierr = ISDuplicate(irows,&icols);CHKERRQ(ierr);
/*get rows information */
ierr = ISGetLocalSize(irows,&irows_ln);CHKERRQ(ierr);
ierr = ISGetIndices(irows,&irows_indices);CHKERRQ(ierr);
/* create a mapping from local to global */
ierr = ISLocalToGlobalMappingCreate(comm,1,irows_ln,irows_indices,PETSC_COPY_VALUES,mapping);CHKERRQ(ierr);
ierr = ISRestoreIndices(irows,&irows_indices);CHKERRQ(ierr);
/* extract a submatrix*/
ierr = MatGetSubMatrices(adj,1,&irows,&icols,MAT_INITIAL_MATRIX,&sadj);CHKERRQ(ierr);
ierr = ISDestroy(&irows);CHKERRQ(ierr);
ierr = ISDestroy(&icols);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
Mat mat,submat,*submatrices;
PetscInt m = 10,n = 10,i = 4,tmp;
PetscErrorCode ierr;
IS irkeep,ickeep;
PetscScalar value = 1.0;
PetscViewer sviewer;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_SELF,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&mat);CHKERRQ(ierr);
ierr = MatSetSizes(mat,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(mat);CHKERRQ(ierr);
for (i=0; i<m; i++) {
value = (PetscReal)i+1; tmp = i % 5;
ierr = MatSetValues(mat,1,&tmp,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"Original matrix\n");CHKERRQ(ierr);
ierr = MatView(mat,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Form submatrix with rows 2-4 and columns 4-8 */
ierr = ISCreateStride(PETSC_COMM_SELF,3,2,1,&irkeep);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,5,4,1,&ickeep);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat,1,&irkeep,&ickeep,MAT_INITIAL_MATRIX,&submatrices);CHKERRQ(ierr);
submat = *submatrices;
ierr = PetscFree(submatrices);CHKERRQ(ierr);
/*
sviewer will cause the submatrices (one per processor) to be printed in the correct order
*/
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"Submatrices\n");CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = MatView(submat,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Zero the original matrix */
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"Original zeroed matrix\n");CHKERRQ(ierr);
ierr = MatZeroEntries(mat);CHKERRQ(ierr);
ierr = MatView(mat,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&irkeep);CHKERRQ(ierr);
ierr = ISDestroy(&ickeep);CHKERRQ(ierr);
ierr = MatDestroy(&submat);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*
DMDAGetFaceInterpolation - Gets the interpolation for a face based coarse space
*/
PetscErrorCode DMDAGetFaceInterpolation(DM da,PC_Exotic *exotic,Mat Aglobal,MatReuse reuse,Mat *P)
{
PetscErrorCode ierr;
PetscInt dim,i,j,k,m,n,p,dof,Nint,Nface,Nwire,Nsurf,*Iint,*Isurf,cint = 0,csurf = 0,istart,jstart,kstart,*II,N,c = 0;
PetscInt mwidth,nwidth,pwidth,cnt,mp,np,pp,Ntotal,gl[6],*globals,Ng,*IIint,*IIsurf,Nt;
Mat Xint, Xsurf,Xint_tmp;
IS isint,issurf,is,row,col;
ISLocalToGlobalMapping ltg;
MPI_Comm comm;
Mat A,Aii,Ais,Asi,*Aholder,iAii;
MatFactorInfo info;
PetscScalar *xsurf,*xint;
#if defined(PETSC_USE_DEBUG_foo)
PetscScalar tmp;
#endif
PetscTable ht;
PetscFunctionBegin;
ierr = DMDAGetInfo(da,&dim,0,0,0,&mp,&np,&pp,&dof,0,0,0,0,0);
CHKERRQ(ierr);
if (dof != 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only for single field problems");
if (dim != 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only coded for 3d problems");
ierr = DMDAGetCorners(da,0,0,0,&m,&n,&p);
CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&istart,&jstart,&kstart,&mwidth,&nwidth,&pwidth);
CHKERRQ(ierr);
istart = istart ? -1 : 0;
jstart = jstart ? -1 : 0;
kstart = kstart ? -1 : 0;
/*
the columns of P are the interpolation of each coarse grid point (one for each vertex and edge)
to all the local degrees of freedom (this includes the vertices, edges and faces).
Xint are the subset of the interpolation into the interior
Xface are the interpolation onto faces but not into the interior
Xsurf are the interpolation onto the vertices and edges (the surfbasket)
Xint
Symbolically one could write P = (Xface) after interchanging the rows to match the natural ordering on the domain
Xsurf
*/
N = (m - istart)*(n - jstart)*(p - kstart);
Nint = (m-2-istart)*(n-2-jstart)*(p-2-kstart);
Nface = 2*((m-2-istart)*(n-2-jstart) + (m-2-istart)*(p-2-kstart) + (n-2-jstart)*(p-2-kstart));
Nwire = 4*((m-2-istart) + (n-2-jstart) + (p-2-kstart)) + 8;
Nsurf = Nface + Nwire;
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nint,6,NULL,&Xint);
CHKERRQ(ierr);
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nsurf,6,NULL,&Xsurf);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/*
Require that all 12 edges and 6 faces have at least one grid point. Otherwise some of the columns of
Xsurf will be all zero (thus making the coarse matrix singular).
*/
if (m-istart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in X direction must be at least 3");
if (n-jstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Y direction must be at least 3");
if (p-kstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Z direction must be at least 3");
cnt = 0;
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 0*Nsurf] = 1;
}
for (k=1; k<p-1-kstart; k++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 1*Nsurf] = 1;
for (j=1; j<n-1-jstart; j++) {
xsurf[cnt++ + 2*Nsurf] = 1;
/* these are the interior nodes */
xsurf[cnt++ + 3*Nsurf] = 1;
}
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 4*Nsurf] = 1;
}
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 5*Nsurf] = 1;
}
#if defined(PETSC_USE_DEBUG_foo)
for (i=0; i<Nsurf; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xsurf[i+j*Nsurf];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xsurf interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
#endif
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/* ierr = MatView(Xsurf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
/*
I are the indices for all the needed vertices (in global numbering)
Iint are the indices for the interior values, I surf for the surface values
(This is just for the part of the global matrix obtained with MatGetSubMatrix(), it
is NOT the local DMDA ordering.)
IIint and IIsurf are the same as the Iint, Isurf except they are in the global numbering
*/
#define Endpoint(a,start,b) (a == 0 || a == (b-1-start))
ierr = PetscMalloc3(N,&II,Nint,&Iint,Nsurf,&Isurf);
CHKERRQ(ierr);
ierr = PetscMalloc2(Nint,&IIint,Nsurf,&IIsurf);
CHKERRQ(ierr);
for (k=0; k<p-kstart; k++) {
for (j=0; j<n-jstart; j++) {
for (i=0; i<m-istart; i++) {
II[c++] = i + j*mwidth + k*mwidth*nwidth;
if (!Endpoint(i,istart,m) && !Endpoint(j,jstart,n) && !Endpoint(k,kstart,p)) {
IIint[cint] = i + j*mwidth + k*mwidth*nwidth;
Iint[cint++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
} else {
IIsurf[csurf] = i + j*mwidth + k*mwidth*nwidth;
Isurf[csurf++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
}
}
}
}
if (c != N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"c != N");
if (cint != Nint) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"cint != Nint");
if (csurf != Nsurf) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"csurf != Nsurf");
ierr = DMGetLocalToGlobalMapping(da,<g);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,N,II,II);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nint,IIint,IIint);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nsurf,IIsurf,IIsurf);
CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)da,&comm);
CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,N,II,PETSC_COPY_VALUES,&is);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nint,Iint,PETSC_COPY_VALUES,&isint);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nsurf,Isurf,PETSC_COPY_VALUES,&issurf);
CHKERRQ(ierr);
ierr = PetscFree3(II,Iint,Isurf);
CHKERRQ(ierr);
ierr = ISSort(is);
CHKERRQ(ierr);
ierr = MatGetSubMatrices(Aglobal,1,&is,&is,MAT_INITIAL_MATRIX,&Aholder);
CHKERRQ(ierr);
A = *Aholder;
ierr = PetscFree(Aholder);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,isint,MAT_INITIAL_MATRIX,&Aii);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,issurf,MAT_INITIAL_MATRIX,&Ais);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,issurf,isint,MAT_INITIAL_MATRIX,&Asi);
CHKERRQ(ierr);
/*
Solve for the interpolation onto the interior Xint
*/
ierr = MatMatMult(Ais,Xsurf,MAT_INITIAL_MATRIX,PETSC_DETERMINE,&Xint_tmp);
CHKERRQ(ierr);
ierr = MatScale(Xint_tmp,-1.0);
CHKERRQ(ierr);
if (exotic->directSolve) {
ierr = MatGetFactor(Aii,MATSOLVERPETSC,MAT_FACTOR_LU,&iAii);
CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);
CHKERRQ(ierr);
ierr = MatGetOrdering(Aii,MATORDERINGND,&row,&col);
CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(iAii,Aii,row,col,&info);
CHKERRQ(ierr);
ierr = ISDestroy(&row);
CHKERRQ(ierr);
ierr = ISDestroy(&col);
CHKERRQ(ierr);
ierr = MatLUFactorNumeric(iAii,Aii,&info);
CHKERRQ(ierr);
ierr = MatMatSolve(iAii,Xint_tmp,Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&iAii);
CHKERRQ(ierr);
} else {
Vec b,x;
PetscScalar *xint_tmp;
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&x);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&b);
CHKERRQ(ierr);
ierr = KSPSetOperators(exotic->ksp,Aii,Aii);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = VecPlaceArray(x,xint+i*Nint);
CHKERRQ(ierr);
ierr = VecPlaceArray(b,xint_tmp+i*Nint);
CHKERRQ(ierr);
ierr = KSPSolve(exotic->ksp,b,x);
CHKERRQ(ierr);
ierr = VecResetArray(x);
CHKERRQ(ierr);
ierr = VecResetArray(b);
CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
}
ierr = MatDestroy(&Xint_tmp);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
for (i=0; i<Nint; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xint[i+j*Nint];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xint interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
/* ierr =MatView(Xint,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
#endif
/* total faces */
Ntotal = mp*np*(pp+1) + mp*pp*(np+1) + np*pp*(mp+1);
/*
For each vertex, edge, face on process (in the same orderings as used above) determine its local number including ghost points
*/
cnt = 0;
{
gl[cnt++] = mwidth+1;
}
{
{
gl[cnt++] = mwidth*nwidth+1;
}
{
gl[cnt++] = mwidth*nwidth + mwidth; /* these are the interior nodes */ gl[cnt++] = mwidth*nwidth + mwidth+m-istart-1;
}
{
gl[cnt++] = mwidth*nwidth+mwidth*(n-jstart-1)+1;
}
}
{
gl[cnt++] = mwidth*nwidth*(p-kstart-1) + mwidth+1;
}
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* convert that to global numbering and get them on all processes */
ierr = ISLocalToGlobalMappingApply(ltg,6,gl,gl);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
ierr = PetscMalloc1(6*mp*np*pp,&globals);
CHKERRQ(ierr);
ierr = MPI_Allgather(gl,6,MPIU_INT,globals,6,MPIU_INT,PetscObjectComm((PetscObject)da));
CHKERRQ(ierr);
/* Number the coarse grid points from 0 to Ntotal */
ierr = MatGetSize(Aglobal,&Nt,NULL);
CHKERRQ(ierr);
ierr = PetscTableCreate(Ntotal/3,Nt+1,&ht);
CHKERRQ(ierr);
for (i=0; i<6*mp*np*pp; i++) {
ierr = PetscTableAddCount(ht,globals[i]+1);
CHKERRQ(ierr);
}
ierr = PetscTableGetCount(ht,&cnt);
CHKERRQ(ierr);
if (cnt != Ntotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Hash table size %D not equal to total number coarse grid points %D",cnt,Ntotal);
ierr = PetscFree(globals);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = PetscTableFind(ht,gl[i]+1,&gl[i]);
CHKERRQ(ierr);
gl[i]--;
}
ierr = PetscTableDestroy(&ht);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* construct global interpolation matrix */
ierr = MatGetLocalSize(Aglobal,&Ng,NULL);
CHKERRQ(ierr);
if (reuse == MAT_INITIAL_MATRIX) {
ierr = MatCreateAIJ(PetscObjectComm((PetscObject)da),Ng,PETSC_DECIDE,PETSC_DECIDE,Ntotal,Nint+Nsurf,NULL,Nint,NULL,P);
CHKERRQ(ierr);
} else {
ierr = MatZeroEntries(*P);
CHKERRQ(ierr);
}
ierr = MatSetOption(*P,MAT_ROW_ORIENTED,PETSC_FALSE);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nint,IIint,6,gl,xint,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nsurf,IIsurf,6,gl,xsurf,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = PetscFree2(IIint,IIsurf);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
{
Vec x,y;
PetscScalar *yy;
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),Ng,PETSC_DETERMINE,&y);
CHKERRQ(ierr);
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),PETSC_DETERMINE,Ntotal,&x);
CHKERRQ(ierr);
ierr = VecSet(x,1.0);
CHKERRQ(ierr);
ierr = MatMult(*P,x,y);
CHKERRQ(ierr);
ierr = VecGetArray(y,&yy);
CHKERRQ(ierr);
for (i=0; i<Ng; i++) {
if (PetscAbsScalar(yy[i]-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong p interpolation at i %D value %g",i,(double)PetscAbsScalar(yy[i]));
}
ierr = VecRestoreArray(y,&yy);
CHKERRQ(ierr);
ierr = VecDestroy(x);
CHKERRQ(ierr);
ierr = VecDestroy(y);
CHKERRQ(ierr);
}
#endif
ierr = MatDestroy(&Aii);
CHKERRQ(ierr);
ierr = MatDestroy(&Ais);
CHKERRQ(ierr);
ierr = MatDestroy(&Asi);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = ISDestroy(&is);
CHKERRQ(ierr);
ierr = ISDestroy(&isint);
CHKERRQ(ierr);
ierr = ISDestroy(&issurf);
CHKERRQ(ierr);
ierr = MatDestroy(&Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&Xsurf);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCGAMGProlongator_Classical_Standard(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
{
PetscErrorCode ierr;
Mat *lA;
Vec lv,v,cv;
PetscScalar *lcid;
IS lis;
PetscInt fs,fe,cs,ce,nl,i,j,k,li,lni,ci;
VecScatter lscat;
PetscInt fn,cn,cid,c_indx;
PetscBool iscoarse;
PetscScalar c_scalar;
const PetscScalar *vcol;
const PetscInt *icol;
const PetscInt *gidx;
PetscInt ncols;
PetscInt *lsparse,*gsparse;
MatType mtype;
PetscInt maxcols;
PetscReal diag,jdiag,jwttotal;
PetscScalar *pvcol,vi;
PetscInt *picol;
PetscInt pncols;
PetscScalar *pcontrib,pentry,pjentry;
/* PC_MG *mg = (PC_MG*)pc->data; */
/* PC_GAMG *gamg = (PC_GAMG*)mg->innerctx; */
PetscFunctionBegin;
ierr = MatGetOwnershipRange(A,&fs,&fe);CHKERRQ(ierr);
fn = fe-fs;
ierr = MatGetVecs(A,NULL,&v);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,fe-fs,fs,1,&lis);CHKERRQ(ierr);
/* increase the overlap by two to get neighbors of neighbors */
ierr = MatIncreaseOverlap(A,1,&lis,2);CHKERRQ(ierr);
ierr = ISSort(lis);CHKERRQ(ierr);
/* get the local part of A */
ierr = MatGetSubMatrices(A,1,&lis,&lis,MAT_INITIAL_MATRIX,&lA);CHKERRQ(ierr);
/* build the scatter out of it */
ierr = ISGetLocalSize(lis,&nl);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,nl,&lv);CHKERRQ(ierr);
ierr = VecScatterCreate(v,lis,lv,NULL,&lscat);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*nl,&pcontrib);CHKERRQ(ierr);
/* create coarse vector */
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse);CHKERRQ(ierr);
if (!iscoarse) {
cn++;
}
}
ierr = VecCreateMPI(PetscObjectComm((PetscObject)A),cn,PETSC_DECIDE,&cv);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(cv,&cs,&ce);CHKERRQ(ierr);
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
if (!iscoarse) {
cid = cs+cn;
cn++;
} else {
cid = -1;
}
*(PetscInt*)&c_scalar = cid;
c_indx = fs+i;
ierr = VecSetValues(v,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecScatterBegin(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* count to preallocate the prolongator */
ierr = ISGetIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecGetArray(lv,&lcid);CHKERRQ(ierr);
maxcols = 0;
/* count the number of unique contributing coarse cells for each fine */
for (i=0;i<nl;i++) {
pcontrib[i] = 0.;
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
lsparse[li] = 0;
gsparse[li] = 0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
lsparse[li] = 1;
} else {
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
pcontrib[icol[j]] = 1.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0) {
pcontrib[icol[k]] = 1.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[j]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[j]] = 0.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[k]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
}
if (lsparse[li] + gsparse[li] > maxcols) maxcols = lsparse[li]+gsparse[li];
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscInt)*maxcols,&picol);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*maxcols,&pvcol);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),P);CHKERRQ(ierr);
ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
for (i=0;i<nl;i++) {
diag = 0.;
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
pncols=0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
pncols = 1;
picol[0] = cid;
pvcol[0] = 1.;
} else {
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
pentry = vcol[j];
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
/* coarse neighbor */
pcontrib[icol[j]] += pentry;
} else if (icol[j] != i) {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
vi = vcol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
jwttotal=0.;
jdiag = 0.;
for (k=0;k<ncols;k++) {
if (ci == icol[k]) {
jdiag = PetscRealPart(vcol[k]);
}
}
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k];
jwttotal += PetscRealPart(pjentry);
}
}
if (jwttotal != 0.) {
jwttotal = PetscRealPart(vi)/jwttotal;
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k]*jwttotal;
pcontrib[icol[k]] += pjentry;
}
}
} else {
diag += PetscRealPart(vi);
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
} else {
diag += PetscRealPart(vcol[j]);
}
}
if (diag != 0.) {
diag = 1./diag;
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
/* the neighbor is a coarse node */
if (PetscAbsScalar(pcontrib[icol[j]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[j]]);
pvcol[pncols] = -pcontrib[icol[j]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[j]] = 0.;
} else {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
if (PetscAbsScalar(pcontrib[icol[k]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[k]]);
pvcol[pncols] = -pcontrib[icol[k]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
pcontrib[icol[j]] = 0.;
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
}
ci = gidx[i];
li = gidx[i] - fs;
if (pncols > 0) {
ierr = MatSetValues(*P,1,&ci,pncols,picol,pvcol,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISRestoreIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecRestoreArray(lv,&lcid);CHKERRQ(ierr);
ierr = PetscFree(pcontrib);CHKERRQ(ierr);
ierr = PetscFree(picol);CHKERRQ(ierr);
ierr = PetscFree(pvcol);CHKERRQ(ierr);
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = ISDestroy(&lis);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lA);CHKERRQ(ierr);
ierr = VecDestroy(&lv);CHKERRQ(ierr);
ierr = VecDestroy(&cv);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = VecScatterDestroy(&lscat);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Mat Pold;
ierr = PCGAMGProlongator_Classical(pc,A,G,agg_lists,&Pold);CHKERRQ(ierr);
ierr = MatView(Pold,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatView(*P,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&Pold);CHKERRQ(ierr);
*/
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscInt bs=1,m=43,ov=1,i,j,k,*rows,*cols,M,nd=5,*idx,mm,nn;
PetscErrorCode ierr;
PetscMPIInt size;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
Vec xx,s1,s2;
PetscReal s1norm,s2norm,rnorm,tol = 1.e-10;
PetscBool flg;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_block_size",&bs,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_size",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-ov",&ov,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-nd",&nd,PETSC_NULL);CHKERRQ(ierr);
/* create a SeqBAIJ matrix A */
M = m*bs;
ierr = MatCreateSeqBAIJ(PETSC_COMM_SELF,bs,M,M,1,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = PetscMalloc(bs*sizeof(PetscInt),&rows);CHKERRQ(ierr);
ierr = PetscMalloc(bs*sizeof(PetscInt),&cols);CHKERRQ(ierr);
ierr = PetscMalloc(bs*bs*sizeof(PetscScalar),&vals);CHKERRQ(ierr);
ierr = PetscMalloc(M*sizeof(PetscScalar),&idx);CHKERRQ(ierr);
/* Now set blocks of random values */
/* first, set diagonal blocks as zero */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<m; i++){
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(int)(PetscRealPart(rval)*m);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = bs*(int)(PetscRealPart(rval)*m);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);
ierr = MatEqual(A, Atrans, &flg);
if (!flg) {
SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
}
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSEQSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
/* Test sA==A through MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(A,&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rand);CHKERRQ(ierr);
ierr = MatMult(A,xx,s1);CHKERRQ(ierr);
ierr = MatMult(sA,xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc(nd*sizeof(IS **),&is1);CHKERRQ(ierr);
ierr = PetscMalloc(nd*sizeof(IS **),&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
size = (int)(PetscRealPart(rval)*m);
for (j=0; j<size; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(int)(PetscRealPart(rval)*m);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,size*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,size*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
}
/* for debugging */
/*
ierr = MatView(A,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatView(sA,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
*/
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
for (i=0; i<nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
ierr = ISSort(is2[i]);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg){
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
SETERRQ1(PETSC_COMM_SELF,1,"i=%d, is1 != is2",i);
}
}
ierr = MatGetSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatGetSubMatrices(sA,nd,is2,is2,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rand);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatsA[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Now test MatGetSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatGetSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatGetSubMatrices(sA,nd,is2,is2,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rand);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatsA[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
ierr = MatDestroy(&submatA[i]);CHKERRQ(ierr);
ierr = MatDestroy(&submatsA[i]);CHKERRQ(ierr);
}
ierr = PetscFree(submatA);CHKERRQ(ierr);
ierr = PetscFree(submatsA);CHKERRQ(ierr);
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat A, *S;
IS rowis[2], colis[2];
PetscInt n,N,i,j,k,l,nsub,Jlow[2] = {0,1}, *jlow, Jhigh[2] = {3,4}, *jhigh, row, col, *subindices, ncols;
const PetscInt *cols;
PetscScalar v;
PetscMPIInt rank, size, p, inversions, total_inversions;
PetscBool sort_rows, sort_cols, show_inversions;
PetscErrorCode ierr;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size>2) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG, "A uniprocessor or two-processor example only.\n");
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
if (size > 1) {
n = 8; N = 16;
} else {
n = 16; N = 16;
}
ierr = MatSetSizes(A,n,n,N,N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/* Don't care if the entries are set multiple times by different procs. */
for (i=0; i<4; ++i) {
for (j = 0; j<4; ++j) {
row = j*4+i;
v = -1.0;
if (i>0) {
col = row-1; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (i<3) {
col = row+1; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>0) {
col = row-4; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (j<3) {
col = row+4; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
v = 4.0;
ierr = MatSetValues(A,1,&row,1,&row,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Original matrix\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (size > 1) {
nsub = 1; /* one subdomain per rank */
}
else {
nsub = 2; /* both subdomains on rank 0 */
}
if (rank) {
jlow = Jlow+1; jhigh = Jhigh+1;
}
else {
jlow = Jlow; jhigh = Jhigh;
}
sort_rows = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-sort_rows", &sort_rows, NULL);CHKERRQ(ierr);
sort_cols = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-sort_cols", &sort_cols, NULL);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
ierr = PetscMalloc1(12, &subindices);CHKERRQ(ierr);
k = 0;
for (i = 0; i < 4; ++i) {
for (j = jlow[l]; j < jhigh[l]; ++j) {
subindices[k] = j*4+i;
k++;
}
}
ierr = ISCreateGeneral(PETSC_COMM_SELF, 12, subindices, PETSC_OWN_POINTER, rowis+l);CHKERRQ(ierr);
if ((sort_rows && !sort_cols) || (!sort_rows && sort_cols)) {
ierr = ISDuplicate(rowis[l],colis+l);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)rowis[l]);CHKERRQ(ierr);
colis[l] = rowis[l];
}
if (sort_rows) {
ierr = ISSort(rowis[l]);CHKERRQ(ierr);
}
if (sort_cols) {
ierr = ISSort(colis[l]);CHKERRQ(ierr);
}
}
ierr = PetscMalloc1(nsub, &S);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,nsub,rowis,colis,MAT_INITIAL_MATRIX, &S);CHKERRQ(ierr);
show_inversions = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-show_inversions", &show_inversions, NULL);CHKERRQ(ierr);
inversions = 0;
for (p = 0; p < size; ++p) {
if (p == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Number of subdomains: %D:\n", rank, size, nsub);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
PetscInt i0, i1;
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Subdomain row IS %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = ISView(rowis[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Subdomain col IS %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = ISView(colis[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Submatrix %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = MatView(S[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (show_inversions) {
ierr = MatGetOwnershipRange(S[l], &i0,&i1);CHKERRQ(ierr);
for (i = i0; i < i1; ++i) {
ierr = MatGetRow(S[l], i, &ncols, &cols, NULL);CHKERRQ(ierr);
for (j = 1; j < ncols; ++j) {
if (cols[j] < cols[j-1]) {
ierr = PetscPrintf(PETSC_COMM_SELF, "***Inversion in row %D: col[%D] = %D < %D = col[%D]\n", i, j, cols[j], cols[j-1], j-1);CHKERRQ(ierr);
inversions++;
}
}
ierr = MatRestoreRow(S[l], i, &ncols, &cols, NULL);CHKERRQ(ierr);
}
}
}
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
if (show_inversions) {
ierr = MPI_Reduce(&inversions,&total_inversions,1,MPIU_INT, MPIU_SUM,0,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "*Total inversions: %D\n", total_inversions);CHKERRQ(ierr);
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
ierr = MatDestroy(&(S[l]));CHKERRQ(ierr);
ierr = ISDestroy(&(rowis[l]));CHKERRQ(ierr);
ierr = ISDestroy(&(colis[l]));CHKERRQ(ierr);
}
ierr = PetscFree(S);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode MatPartitioningApply_Chaco(MatPartitioning part,IS *partitioning)
{
PetscErrorCode ierr;
PetscInt *parttab,*locals,i,nb_locals,M,N;
PetscMPIInt size,rank;
Mat mat = part->adj,matAdj,matSeq,*A;
Mat_MPIAdj *adj;
MatPartitioning_Chaco *chaco = (MatPartitioning_Chaco*)part->data;
PetscBool flg;
IS isrow, iscol;
int nvtxs,*start,*adjacency,*vwgts,architecture,ndims_tot;
int mesh_dims[3],global_method,local_method,rqi_flag,vmax,ndims;
short *assignment;
double eigtol;
long seed;
char *mesg_log;
#if defined(PETSC_HAVE_UNISTD_H)
int fd_stdout,fd_pipe[2],count,err;
#endif
PetscFunctionBegin;
FREE_GRAPH = 0; /* otherwise Chaco will attempt to free memory for adjacency graph */
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)mat),&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (size>1) {
if (flg) SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Distributed matrix format MPIAdj is not supported for sequential partitioners");
ierr = PetscInfo(part,"Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);
ierr = MatGetSize(mat,&M,&N);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,N,0,1,&iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat,1,&isrow,&iscol,MAT_INITIAL_MATRIX,&A);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
matSeq = *A;
ierr = PetscFree(A);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)mat);CHKERRQ(ierr);
matSeq = mat;
}
if (!flg) { /* convert regular matrix to MPIADJ */
ierr = MatConvert(matSeq,MATMPIADJ,MAT_INITIAL_MATRIX,&matAdj);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)matSeq);CHKERRQ(ierr);
matAdj = matSeq;
}
adj = (Mat_MPIAdj*)matAdj->data; /* finaly adj contains adjacency graph */
/* arguments for Chaco library */
nvtxs = mat->rmap->N; /* number of vertices in full graph */
start = adj->i; /* start of edge list for each vertex */
vwgts = part->vertex_weights; /* weights for all vertices */
architecture = 1; /* 0 => hypercube, d => d-dimensional mesh */
ndims_tot = 0; /* total number of cube dimensions to divide */
mesh_dims[0] = part->n; /* dimensions of mesh of processors */
global_method = chaco->global_method; /* global partitioning algorithm */
local_method = chaco->local_method; /* local partitioning algorithm */
rqi_flag = chaco->eigen_method; /* should I use RQI/Symmlq eigensolver? */
vmax = chaco->nbvtxcoarsed; /* how many vertices to coarsen down to? */
ndims = chaco->eignum; /* number of eigenvectors (2^d sets) */
eigtol = chaco->eigtol; /* tolerance on eigenvectors */
seed = 123636512; /* for random graph mutations */
ierr = PetscMalloc((mat->rmap->N)*sizeof(short),&assignment);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(int)*start[nvtxs],&adjacency);CHKERRQ(ierr);
for (i=0; i<start[nvtxs]; i++) adjacency[i] = (adj->j)[i] + 1; /* 1-based indexing */
/* redirect output to buffer */
#if defined(PETSC_HAVE_UNISTD_H)
fd_stdout = dup(1);
if (pipe(fd_pipe)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"Could not open pipe");
close(1);
dup2(fd_pipe[1],1);
ierr = PetscMalloc(SIZE_LOG*sizeof(char),&mesg_log);CHKERRQ(ierr);
#endif
/* library call */
ierr = interface(nvtxs,start,adjacency,vwgts,NULL,NULL,NULL,NULL,
NULL,NULL,assignment,architecture,ndims_tot,mesh_dims,
NULL,global_method,local_method,rqi_flag,vmax,ndims,eigtol,seed);
#if defined(PETSC_HAVE_UNISTD_H)
err = fflush(stdout);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on stdout");
count = read(fd_pipe[0],mesg_log,(SIZE_LOG-1)*sizeof(char));
if (count<0) count = 0;
mesg_log[count] = 0;
close(1);
dup2(fd_stdout,1);
close(fd_stdout);
close(fd_pipe[0]);
close(fd_pipe[1]);
if (chaco->verbose) {
ierr = PetscPrintf(PetscObjectComm((PetscObject)mat),mesg_log);
}
ierr = PetscFree(mesg_log);CHKERRQ(ierr);
#endif
if (ierr) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Chaco failed");
ierr = PetscMalloc((mat->rmap->N)*sizeof(PetscInt),&parttab);CHKERRQ(ierr);
for (i=0; i<nvtxs; i++) parttab[i] = assignment[i];
/* creation of the index set */
nb_locals = mat->rmap->N / size;
locals = parttab + rank*nb_locals;
if (rank < mat->rmap->N % size) {
nb_locals++;
locals += rank;
} else locals += mat->rmap->N % size;
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)part),nb_locals,locals,PETSC_COPY_VALUES,partitioning);CHKERRQ(ierr);
/* clean up */
ierr = PetscFree(parttab);CHKERRQ(ierr);
ierr = PetscFree(adjacency);CHKERRQ(ierr);
ierr = PetscFree(assignment);CHKERRQ(ierr);
ierr = MatDestroy(&matSeq);CHKERRQ(ierr);
ierr = MatDestroy(&matAdj);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatColoringGetDegrees(Mat G,PetscInt distance,PetscInt *degrees)
{
PetscInt j,i,s,e,n,ln,lm,degree,bidx,idx,dist;
Mat lG,*lGs;
IS ris;
PetscErrorCode ierr;
PetscInt *seen;
const PetscInt *gidx;
PetscInt *idxbuf;
PetscInt *distbuf;
PetscInt ncols;
const PetscInt *cols;
PetscBool isSEQAIJ;
Mat_SeqAIJ *aij;
PetscInt *Gi,*Gj;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(G,&s,&e);CHKERRQ(ierr);
n=e-s;
ierr = ISCreateStride(PetscObjectComm((PetscObject)G),n,s,1,&ris);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(G,1,&ris,distance);CHKERRQ(ierr);
ierr = ISSort(ris);CHKERRQ(ierr);
ierr = MatGetSubMatrices(G,1,&ris,&ris,MAT_INITIAL_MATRIX,&lGs);CHKERRQ(ierr);
lG = lGs[0];
ierr = PetscObjectTypeCompare((PetscObject)lG,MATSEQAIJ,&isSEQAIJ);CHKERRQ(ierr);
if (!isSEQAIJ) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_SUP,"Requires an MPI/SEQAIJ Matrix");
ierr = MatGetSize(lG,&ln,&lm);CHKERRQ(ierr);
aij = (Mat_SeqAIJ*)lG->data;
Gi = aij->i;
Gj = aij->j;
ierr = PetscMalloc3(lm,&seen,lm,&idxbuf,lm,&distbuf);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
seen[i]=-1;
}
ierr = ISGetIndices(ris,&gidx);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
if (gidx[i] >= e || gidx[i] < s) continue;
bidx=-1;
ncols = Gi[i+1]-Gi[i];
cols = &(Gj[Gi[i]]);
degree = 0;
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
degree++;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
degrees[gidx[i]-s] = degree;
}
ierr = ISRestoreIndices(ris,&gidx);CHKERRQ(ierr);
ierr = ISDestroy(&ris);CHKERRQ(ierr);
ierr = PetscFree3(seen,idxbuf,distbuf);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lGs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode MatPartitioningApply_Scotch(MatPartitioning part, IS * partitioning)
{
PetscErrorCode ierr;
int *parttab, *locals = PETSC_NULL, rank, i, size;
size_t j;
Mat mat = part->adj, matMPI, matSeq;
int nb_locals = mat->rmap->n;
Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
PetscTruth flg;
#ifdef PETSC_HAVE_UNISTD_H
int fd_stdout, fd_pipe[2], count,err;
#endif
PetscFunctionBegin;
/* check if the matrix is sequential, use MatGetSubMatrices if necessary */
ierr = MPI_Comm_size(((PetscObject)mat)->comm, &size);CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);CHKERRQ(ierr);
if (size > 1) {
int M, N;
IS isrow, iscol;
Mat *A;
if (flg) {
SETERRQ(0, "Distributed matrix format MPIAdj is not supported for sequential partitioners");
}
PetscPrintf(((PetscObject)part)->comm, "Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);
ierr = MatGetSize(mat, &M, &N);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);CHKERRQ(ierr);
matSeq = *A;
ierr = PetscFree(A);CHKERRQ(ierr);
ierr = ISDestroy(isrow);CHKERRQ(ierr);
ierr = ISDestroy(iscol);CHKERRQ(ierr);
} else
matSeq = mat;
/* convert the the matrix to MPIADJ type if necessary */
if (!flg) {
ierr = MatConvert(matSeq, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);CHKERRQ(ierr);
} else {
matMPI = matSeq;
}
adj = (Mat_MPIAdj *) matMPI->data; /* finaly adj contains adjacency graph */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
{
/* definition of Scotch library arguments */
SCOTCH_Strat stratptr; /* scotch strategy */
SCOTCH_Graph grafptr; /* scotch graph */
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
int vertnbr = mat->rmap->N; /* number of vertices in full graph */
int *verttab = adj->i; /* start of edge list for each vertex */
int *edgetab = adj->j; /* edge list data */
int edgenbr = adj->nz; /* number of edges */
int *velotab = NULL; /* not used by petsc interface */
int *vlbltab = NULL;
int *edlotab = NULL;
int flagval = 3; /* (cf doc scotch no weight edge & vertices) */
#endif
int baseval = 0; /* 0 for C array indexing */
char strategy[256];
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab);CHKERRQ(ierr);
/* redirect output to buffer scotch -> mesg_log */
#ifdef PETSC_HAVE_UNISTD_H
fd_stdout = dup(1);
pipe(fd_pipe);
close(1);
dup2(fd_pipe[1], 1);
ierr = PetscMalloc(SIZE_LOG * sizeof(char), &(scotch->mesg_log));CHKERRQ(ierr);
#endif
/* library call */
/* Construction of the scotch graph object */
ierr = SCOTCH_graphInit(&grafptr);
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
ierr = SCOTCH_graphBuild((SCOTCH_Graph *) &grafptr,
(const SCOTCH_Num) vertnbr,
(const SCOTCH_Num) verttab,
(const SCOTCH_Num *)velotab,
(const SCOTCH_Num *)vlbltab,
(const SCOTCH_Num *)edgenbr,
(const SCOTCH_Num *)edgetab,
(const SCOTCH_Num) edlotab,
(const SCOTCH_Num *)baseval,
(const SCOTCH_Num *)flagval);CHKERRQ(ierr);
#else
SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
ierr = SCOTCH_graphCheck(&grafptr);CHKERRQ(ierr);
/* Construction of the strategy */
if (scotch->strategy[0] != 0) {
ierr = PetscStrcpy(strategy, scotch->strategy);CHKERRQ(ierr);
} else {
PetscStrcpy(strategy, "b{strat=");
if (scotch->multilevel) {
/* PetscStrcat(strategy,"m{vert=");
sprintf(strategy+strlen(strategy),"%d",scotch->nbvtxcoarsed);
PetscStrcat(strategy,",asc="); */
sprintf(strategy, "b{strat=m{vert=%d,asc=",
scotch->nbvtxcoarsed);
} else
PetscStrcpy(strategy, "b{strat=");
switch (scotch->global_method) {
case MP_SCOTCH_GREEDY:
PetscStrcat(strategy, "h");
break;
case MP_SCOTCH_GPS:
PetscStrcat(strategy, "g");
break;
case MP_SCOTCH_GR_GPS:
PetscStrcat(strategy, "g|h");
}
switch (scotch->local_method) {
case MP_SCOTCH_KERNIGHAN_LIN:
if (scotch->multilevel)
PetscStrcat(strategy, ",low=f}");
else
PetscStrcat(strategy, " f");
break;
case MP_SCOTCH_NONE:
if (scotch->multilevel)
PetscStrcat(strategy, ",asc=x}");
default:
break;
}
PetscStrcat(strategy, " x}");
}
PetscPrintf(((PetscObject)part)->comm, "strategy=[%s]\n", strategy);
ierr = SCOTCH_stratInit(&stratptr);CHKERRQ(ierr);
/*
TODO: Correct this part
Commented because this doesn't exists anymore
ierr = SCOTCH_stratMap(&stratptr, strategy);CHKERRQ(ierr);
*/
/* check for option mapping */
if (!scotch->map) {
/* ********************************************
* *
* TODO: Correct this part *
* *
* Won't work with this tmp SCOTCH_Strat... *
* *
* I just modified it to make scotch compile, *
* to be able to use PaStiX... *
* *
**********************************************/
#if defined (DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
SCOTCH_Strat tmp;
ierr = SCOTCH_graphPart((const SCOTCH_Graph *)&grafptr,
(const SCOTCH_Num) &stratptr,
(const SCOTCH_Strat *)&tmp, /* The Argument changed from scotch 3.04 it was part->n, */
(SCOTCH_Num *) parttab);CHKERRQ(ierr);
#else
SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
ierr = PetscPrintf(PETSC_COMM_SELF, "Partition simple without mapping\n");
} else {
SCOTCH_Graph grafarch;
SCOTCH_Num *listtab;
SCOTCH_Num listnbr = 0;
SCOTCH_Arch archptr; /* file in scotch architecture format */
SCOTCH_Strat archstrat;
int arch_total_size, *parttab_tmp,err;
int cpt;
char buf[256];
FILE *file1, *file2;
char host_buf[256];
/* generate the graph that represents the arch */
file1 = fopen(scotch->arch, "r");
if (!file1) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open architecture file %s", scotch->arch);
ierr = SCOTCH_graphInit(&grafarch);CHKERRQ(ierr);
ierr = SCOTCH_graphLoad(&grafarch, file1, baseval, 3);CHKERRQ(ierr);
ierr = SCOTCH_graphCheck(&grafarch);CHKERRQ(ierr);
SCOTCH_graphSize(&grafarch, &arch_total_size, &cpt);
err = fclose(file1);
if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");
printf("total size = %d\n", arch_total_size);
/* generate the list of nodes currently working */
ierr = PetscGetHostName(host_buf, 256);CHKERRQ(ierr);
ierr = PetscStrlen(host_buf, &j);CHKERRQ(ierr);
file2 = fopen(scotch->host_list, "r");
if (!file2) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open host list file %s", scotch->host_list);
i = -1;
flg = PETSC_FALSE;
while (!feof(file2) && !flg) {
i++;
fgets(buf, 256, file2);
PetscStrncmp(buf, host_buf, j, &flg);
}
err = fclose(file2);
if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");
if (!flg) SETERRQ1(PETSC_ERR_LIB, "Scotch: unable to find '%s' in host list file", host_buf);
listnbr = size;
ierr = PetscMalloc(sizeof(SCOTCH_Num) * listnbr, &listtab);CHKERRQ(ierr);
ierr = MPI_Allgather(&i, 1, MPI_INT, listtab, 1, MPI_INT, ((PetscObject)part)->comm);CHKERRQ(ierr);
printf("listnbr = %d, listtab = ", listnbr);
for (i = 0; i < listnbr; i++)
printf("%d ", listtab[i]);
printf("\n");
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on file");
ierr = SCOTCH_stratInit(&archstrat);CHKERRQ(ierr);
/**************************************************************
* *
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* *
* ierr = SCOTCH_stratBipart(&archstrat, "fx");CHKERRQ(ierr); *
**************************************************************/
ierr = SCOTCH_archInit(&archptr);CHKERRQ(ierr);
ierr = SCOTCH_archBuild(&archptr, &grafarch, listnbr, listtab,
&archstrat);CHKERRQ(ierr);
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab_tmp);CHKERRQ(ierr);
/************************************************************************************
* *
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* *
* ierr = SCOTCH_mapInit(&mappptr, &grafptr, &archptr, parttab_tmp);CHKERRQ(ierr); *
* *
* ierr = SCOTCH_mapCompute(&mappptr, &stratptr);CHKERRQ(ierr); *
* *
* ierr = SCOTCH_mapView(&mappptr, stdout);CHKERRQ(ierr); *
************************************************************************************/
/* now we have to set in the real parttab at the good place */
/* because the ranks order are different than position in */
/* the arch graph */
for (i = 0; i < mat->rmap->N; i++) {
parttab[i] = parttab_tmp[i];
}
ierr = PetscFree(listtab);CHKERRQ(ierr);
SCOTCH_archExit(&archptr);
/*************************************************
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* SCOTCH_mapExit(&mappptr); *
*************************************************/
SCOTCH_stratExit(&archstrat);
}
/* dump to mesg_log... */
#ifdef PETSC_HAVE_UNISTD_H
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on stdout");
count = read(fd_pipe[0], scotch->mesg_log, (SIZE_LOG - 1) * sizeof(char));
if (count < 0)
count = 0;
scotch->mesg_log[count] = 0;
close(1);
dup2(fd_stdout, 1);
close(fd_stdout);
close(fd_pipe[0]);
close(fd_pipe[1]);
#endif
SCOTCH_graphExit(&grafptr);
SCOTCH_stratExit(&stratptr);
}
if (ierr)
SETERRQ(PETSC_ERR_LIB, scotch->mesg_log);
/* Creation of the index set */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)part)->comm, &size);CHKERRQ(ierr);
nb_locals = mat->rmap->N / size;
locals = parttab + rank * nb_locals;
if (rank < mat->rmap->N % size) {
nb_locals++;
locals += rank;
} else
locals += mat->rmap->N % size;
ierr = ISCreateGeneral(((PetscObject)part)->comm, nb_locals, locals, partitioning);CHKERRQ(ierr);
/* destroying old objects */
ierr = PetscFree(parttab);CHKERRQ(ierr);
if (matSeq != mat) {
ierr = MatDestroy(matSeq);CHKERRQ(ierr);
}
if (matMPI != mat) {
ierr = MatDestroy(matMPI);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode MatPartitioningApply_Party(MatPartitioning part,IS *partitioning)
{
PetscErrorCode ierr;
PetscInt i,*parttab,*locals,nb_locals,M,N;
PetscMPIInt size,rank;
Mat mat = part->adj,matAdj,matSeq,*A;
Mat_MPIAdj *adj;
MatPartitioning_Party *party = (MatPartitioning_Party*)part->data;
PetscBool flg;
IS isrow, iscol;
int n,*edge_p,*edge,*vertex_w,p,*part_party,cutsize,redl,rec;
const char *redm,*redo;
char *mesg_log;
#if defined(PETSC_HAVE_UNISTD_H)
int fd_stdout,fd_pipe[2],count,err;
#endif
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)mat),&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (size>1) {
if (flg) SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Distributed matrix format MPIAdj is not supported for sequential partitioners");
ierr = PetscInfo(part,"Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);
ierr = MatGetSize(mat,&M,&N);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,N,0,1,&iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat,1,&isrow,&iscol,MAT_INITIAL_MATRIX,&A);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
matSeq = *A;
ierr = PetscFree(A);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)mat);CHKERRQ(ierr);
matSeq = mat;
}
if (!flg) { /* convert regular matrix to MPIADJ */
ierr = MatConvert(matSeq,MATMPIADJ,MAT_INITIAL_MATRIX,&matAdj);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)matSeq);CHKERRQ(ierr);
matAdj = matSeq;
}
adj = (Mat_MPIAdj*)matAdj->data; /* finaly adj contains adjacency graph */
/* arguments for Party library */
n = mat->rmap->N; /* number of vertices in full graph */
edge_p = adj->i; /* start of edge list for each vertex */
edge = adj->j; /* edge list data */
vertex_w = part->vertex_weights; /* weights for all vertices */
p = part->n; /* number of parts to create */
redl = party->nbvtxcoarsed; /* how many vertices to coarsen down to? */
rec = party->recursive ? 1 : 0; /* recursive bisection */
redm = party->redm ? "lam" : ""; /* matching method */
redo = party->redo ? "w3" : ""; /* matching optimization method */
ierr = PetscMalloc1(mat->rmap->N,&part_party);CHKERRQ(ierr);
/* redirect output to buffer */
#if defined(PETSC_HAVE_UNISTD_H)
fd_stdout = dup(1);
if (pipe(fd_pipe)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"Could not open pipe");
close(1);
dup2(fd_pipe[1],1);
ierr = PetscMalloc1(SIZE_LOG,&mesg_log);CHKERRQ(ierr);
#endif
/* library call */
party_lib_times_start();
ierr = party_lib(n,vertex_w,NULL,NULL,NULL,edge_p,edge,NULL,p,part_party,&cutsize,redl,(char*)redm,(char*)redo,party->global,party->local,rec,1);
party_lib_times_output(1);
part_info(n,vertex_w,edge_p,edge,NULL,p,part_party,1);
#if defined(PETSC_HAVE_UNISTD_H)
err = fflush(stdout);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on stdout");
count = read(fd_pipe[0],mesg_log,(SIZE_LOG-1)*sizeof(char));
if (count<0) count = 0;
mesg_log[count] = 0;
close(1);
dup2(fd_stdout,1);
close(fd_stdout);
close(fd_pipe[0]);
close(fd_pipe[1]);
if (party->verbose) {
ierr = PetscPrintf(PetscObjectComm((PetscObject)mat),mesg_log);
}
ierr = PetscFree(mesg_log);CHKERRQ(ierr);
#endif
if (ierr) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Party failed");
ierr = PetscMalloc1(mat->rmap->N,&parttab);CHKERRQ(ierr);
for (i=0; i<mat->rmap->N; i++) parttab[i] = part_party[i];
/* creation of the index set */
nb_locals = mat->rmap->N / size;
locals = parttab + rank*nb_locals;
if (rank < mat->rmap->N % size) {
nb_locals++;
locals += rank;
} else locals += mat->rmap->N % size;
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)part),nb_locals,locals,PETSC_COPY_VALUES,partitioning);CHKERRQ(ierr);
/* clean up */
ierr = PetscFree(parttab);CHKERRQ(ierr);
ierr = PetscFree(part_party);CHKERRQ(ierr);
ierr = MatDestroy(&matSeq);CHKERRQ(ierr);
ierr = MatDestroy(&matAdj);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
PetscBool flg,TestOverlap,TestSubMat,TestAllcols;
PetscInt vid = -1;
#if defined(PETSC_USE_LOG)
PetscLogStage stages[2];
#endif
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);
ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<Mbs; i++) {
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);
ierr = MatEqual(A, Atrans, &flg);
if (flg) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
if (vid >= 0 && vid < size) {
if (!rank) printf("A: \n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("sA: \n");
ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test sA==A through MatMult() */
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA");
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
if (!TestAllcols) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
if (rank == vid) {
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
} else { /* Test all rows and colums */
sz = M;
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr);
if (rank == vid) {
PetscBool colflag;
ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr);
printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
}
}
ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);
ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);
/* Test MatIncreaseOverlap */
if (TestOverlap) {
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (rank == vid) {
printf("\n[%d] IS from BAIJ:\n",rank);
ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
printf("\n[%d] IS from SBAIJ:\n",rank);
ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
if (!rank) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
ierr = ISSort(is2[i]);CHKERRQ(ierr);
/* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
}
SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i);
}
}
}
/* Test MatGetSubmatrices */
if (TestSubMat) {
for (i = 0; i < nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
ierr = ISSort(is2[i]);CHKERRQ(ierr);
}
ierr = MatGetSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatGetSubMatrices(sA,nd,is2,is2,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA");
/* Now test MatGetSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatGetSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatGetSubMatrices(sA,nd,is2,is2,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatGetSubmatrices(): A != sA");
for (i=0; i<nd; ++i) {
ierr = MatDestroy(&submatA[i]);CHKERRQ(ierr);
ierr = MatDestroy(&submatsA[i]);CHKERRQ(ierr);
}
ierr = PetscFree(submatA);CHKERRQ(ierr);
ierr = PetscFree(submatsA);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*
GetElasticityMatrix - Forms 3D linear elasticity matrix.
*/
PetscErrorCode GetElasticityMatrix(PetscInt m,Mat *newmat)
{
PetscInt i,j,k,i1,i2,j_1,j2,k1,k2,h1,h2,shiftx,shifty,shiftz;
PetscInt ict,nz,base,r1,r2,N,*rowkeep,nstart;
PetscErrorCode ierr;
IS iskeep;
PetscReal **K,norm;
Mat mat,submat = 0,*submatb;
MatType type = MATSEQBAIJ;
m /= 2; /* This is done just to be consistent with the old example */
N = 3*(2*m+1)*(2*m+1)*(2*m+1);
ierr = PetscPrintf(PETSC_COMM_SELF,"m = %D, N=%D\n",m,N);CHKERRQ(ierr);
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,N,N,80,PETSC_NULL,&mat);CHKERRQ(ierr);
/* Form stiffness for element */
ierr = PetscMalloc(81*sizeof(PetscReal *),&K);CHKERRQ(ierr);
for (i=0; i<81; i++) {
ierr = PetscMalloc(81*sizeof(PetscReal),&K[i]);CHKERRQ(ierr);
}
ierr = Elastic20Stiff(K);CHKERRQ(ierr);
/* Loop over elements and add contribution to stiffness */
shiftx = 3; shifty = 3*(2*m+1); shiftz = 3*(2*m+1)*(2*m+1);
for (k=0; k<m; k++) {
for (j=0; j<m; j++) {
for (i=0; i<m; i++) {
h1 = 0;
base = 2*k*shiftz + 2*j*shifty + 2*i*shiftx;
for (k1=0; k1<3; k1++) {
for (j_1=0; j_1<3; j_1++) {
for (i1=0; i1<3; i1++) {
h2 = 0;
r1 = base + i1*shiftx + j_1*shifty + k1*shiftz;
for (k2=0; k2<3; k2++) {
for (j2=0; j2<3; j2++) {
for (i2=0; i2<3; i2++) {
r2 = base + i2*shiftx + j2*shifty + k2*shiftz;
ierr = AddElement(mat,r1,r2,K,h1,h2);CHKERRQ(ierr);
h2 += 3;
}
}
}
h1 += 3;
}
}
}
}
}
}
for (i=0; i<81; i++) {
ierr = PetscFree(K[i]);CHKERRQ(ierr);
}
ierr = PetscFree(K);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Exclude any superfluous rows and columns */
nstart = 3*(2*m+1)*(2*m+1);
ict = 0;
ierr = PetscMalloc((N-nstart)*sizeof(PetscInt),&rowkeep);CHKERRQ(ierr);
for (i=nstart; i<N; i++) {
ierr = MatGetRow(mat,i,&nz,0,0);CHKERRQ(ierr);
if (nz) rowkeep[ict++] = i;
ierr = MatRestoreRow(mat,i,&nz,0,0);CHKERRQ(ierr);
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,ict,rowkeep,PETSC_COPY_VALUES,&iskeep);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat,1,&iskeep,&iskeep,MAT_INITIAL_MATRIX,&submatb);CHKERRQ(ierr);
submat = *submatb;
ierr = PetscFree(submatb);CHKERRQ(ierr);
ierr = PetscFree(rowkeep);CHKERRQ(ierr);
ierr = ISDestroy(&iskeep);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
/* Convert storage formats -- just to demonstrate conversion to various
formats (in particular, block diagonal storage). This is NOT the
recommended means to solve such a problem. */
ierr = MatConvert(submat,type,MAT_INITIAL_MATRIX,newmat);CHKERRQ(ierr);
ierr = MatDestroy(&submat);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = MatView(*newmat,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatNorm(*newmat,NORM_1,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"matrix 1 norm = %G\n",norm);CHKERRQ(ierr);
return 0;
}
PetscErrorCode MatFactorNumeric_PaStiX(Mat F,Mat A,const MatFactorInfo *info)
{
Mat_Pastix *lu =(Mat_Pastix*)(F)->spptr;
Mat *tseq;
PetscErrorCode ierr = 0;
PetscInt icntl;
PetscInt M=A->rmap->N;
PetscBool valOnly,flg, isSym;
Mat F_diag;
IS is_iden;
Vec b;
IS isrow;
PetscBool isSeqAIJ,isSeqSBAIJ,isMPIAIJ;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&isMPIAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isSeqSBAIJ);CHKERRQ(ierr);
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN) {
(F)->ops->solve = MatSolve_PaStiX;
/* Initialize a PASTIX instance */
ierr = MPI_Comm_dup(PetscObjectComm((PetscObject)A),&(lu->pastix_comm));CHKERRQ(ierr);
ierr = MPI_Comm_rank(lu->pastix_comm, &lu->commRank);CHKERRQ(ierr);
ierr = MPI_Comm_size(lu->pastix_comm, &lu->commSize);CHKERRQ(ierr);
/* Set pastix options */
lu->iparm[IPARM_MODIFY_PARAMETER] = API_NO;
lu->iparm[IPARM_START_TASK] = API_TASK_INIT;
lu->iparm[IPARM_END_TASK] = API_TASK_INIT;
lu->rhsnbr = 1;
/* Call to set default pastix options */
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
ierr = PetscOptionsBegin(PetscObjectComm((PetscObject)A),((PetscObject)A)->prefix,"PaStiX Options","Mat");CHKERRQ(ierr);
icntl = -1;
lu->iparm[IPARM_VERBOSE] = API_VERBOSE_NOT;
ierr = PetscOptionsInt("-mat_pastix_verbose","iparm[IPARM_VERBOSE] : level of printing (0 to 2)","None",lu->iparm[IPARM_VERBOSE],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl >= 0) || PetscLogPrintInfo) {
lu->iparm[IPARM_VERBOSE] = icntl;
}
icntl=-1;
ierr = PetscOptionsInt("-mat_pastix_threadnbr","iparm[IPARM_THREAD_NBR] : Number of thread by MPI node","None",lu->iparm[IPARM_THREAD_NBR],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl > 0)) {
lu->iparm[IPARM_THREAD_NBR] = icntl;
}
PetscOptionsEnd();
valOnly = PETSC_FALSE;
} else {
if (isSeqAIJ || isMPIAIJ) {
ierr = PetscFree(lu->colptr);CHKERRQ(ierr);
ierr = PetscFree(lu->row);CHKERRQ(ierr);
ierr = PetscFree(lu->val);CHKERRQ(ierr);
valOnly = PETSC_FALSE;
} else valOnly = PETSC_TRUE;
}
lu->iparm[IPARM_MATRIX_VERIFICATION] = API_YES;
/* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatConvertToCSC(*tseq,valOnly, &lu->n, &lu->colptr, &lu->row, &lu->val);CHKERRQ(ierr);
ierr = MatIsSymmetric(*tseq,0.0,&isSym);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&tseq);CHKERRQ(ierr);
if (!lu->perm) {
ierr = PetscMalloc1((lu->n),&(lu->perm));CHKERRQ(ierr);
ierr = PetscMalloc1((lu->n),&(lu->invp));CHKERRQ(ierr);
}
if (isSym) {
/* On symmetric matrix, LLT */
lu->iparm[IPARM_SYM] = API_SYM_YES;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LDLT;
} else {
/* On unsymmetric matrix, LU */
lu->iparm[IPARM_SYM] = API_SYM_NO;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LU;
}
/*----------------*/
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN) {
if (!(isSeqAIJ || isSeqSBAIJ)) {
/* PaStiX only supports centralized rhs. Create scatter scat_rhs for repeated use in MatSolve() */
ierr = VecCreateSeq(PETSC_COMM_SELF,A->cmap->N,&lu->b_seq);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,A->cmap->N,0,1,&is_iden);CHKERRQ(ierr);
ierr = MatCreateVecs(A,NULL,&b);CHKERRQ(ierr);
ierr = VecScatterCreate(b,is_iden,lu->b_seq,is_iden,&lu->scat_rhs);CHKERRQ(ierr);
ierr = VecScatterCreate(lu->b_seq,is_iden,b,is_iden,&lu->scat_sol);CHKERRQ(ierr);
ierr = ISDestroy(&is_iden);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
}
lu->iparm[IPARM_START_TASK] = API_TASK_ORDERING;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
} else {
lu->iparm[IPARM_START_TASK] = API_TASK_NUMFACT;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
}
if (lu->commSize > 1) {
if ((F)->factortype == MAT_FACTOR_LU) {
F_diag = ((Mat_MPIAIJ*)(F)->data)->A;
} else {
F_diag = ((Mat_MPISBAIJ*)(F)->data)->A;
}
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
lu->matstruc = SAME_NONZERO_PATTERN;
lu->CleanUpPastix = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode MatLUFactorNumeric_SuperLU_DIST(Mat F,Mat A,const MatFactorInfo *info)
{
Mat *tseq,A_seq = NULL;
Mat_SeqAIJ *aa,*bb;
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)(F)->spptr;
PetscErrorCode ierr;
PetscInt M=A->rmap->N,N=A->cmap->N,i,*ai,*aj,*bi,*bj,nz,rstart,*garray,
m=A->rmap->n, colA_start,j,jcol,jB,countA,countB,*bjj,*ajj;
int sinfo; /* SuperLU_Dist info flag is always an int even with long long indices */
PetscMPIInt size;
SuperLUStat_t stat;
double *berr=0;
IS isrow;
Mat F_diag=NULL;
#if defined(PETSC_USE_COMPLEX)
doublecomplex *av, *bv;
#else
double *av, *bv;
#endif
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&size);CHKERRQ(ierr);
if (lu->MatInputMode == GLOBAL) { /* global mat input */
if (size > 1) { /* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
A_seq = *tseq;
ierr = PetscFree(tseq);CHKERRQ(ierr);
aa = (Mat_SeqAIJ*)A_seq->data;
} else {
PetscBool flg;
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&flg);CHKERRQ(ierr);
if (flg) {
Mat_MPIAIJ *At = (Mat_MPIAIJ*)A->data;
A = At->A;
}
aa = (Mat_SeqAIJ*)A->data;
}
/* Convert Petsc NR matrix to SuperLU_DIST NC.
Note: memories of lu->val, col and row are allocated by CompRow_to_CompCol_dist()! */
if (lu->options.Fact != DOFACT) {/* successive numeric factorization, sparsity pattern is reused. */
PetscStackCall("SuperLU_DIST:Destroy_CompCol_Matrix_dist",Destroy_CompCol_Matrix_dist(&lu->A_sup));
if (lu->FactPattern == SamePattern_SameRowPerm) {
lu->options.Fact = SamePattern_SameRowPerm; /* matrix has similar numerical values */
} else { /* lu->FactPattern == SamePattern */
PetscStackCall("SuperLU_DIST:Destroy_LU",Destroy_LU(N, &lu->grid, &lu->LUstruct));
lu->options.Fact = SamePattern;
}
}
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCompRow_to_CompCol_dist",zCompRow_to_CompCol_dist(M,N,aa->nz,(doublecomplex*)aa->a,(int_t*)aa->j,(int_t*)aa->i,&lu->val,&lu->col, &lu->row));
#else
PetscStackCall("SuperLU_DIST:dCompRow_to_CompCol_dist",dCompRow_to_CompCol_dist(M,N,aa->nz,aa->a,(int_t*)aa->j,(int_t*)aa->i,&lu->val, &lu->col, &lu->row));
#endif
/* Create compressed column matrix A_sup. */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCreate_CompCol_Matrix_dist",zCreate_CompCol_Matrix_dist(&lu->A_sup, M, N, aa->nz, lu->val, lu->col, lu->row, SLU_NC, SLU_Z, SLU_GE));
#else
PetscStackCall("SuperLU_DIST:dCreate_CompCol_Matrix_dist",dCreate_CompCol_Matrix_dist(&lu->A_sup, M, N, aa->nz, lu->val, lu->col, lu->row, SLU_NC, SLU_D, SLU_GE));
#endif
} else { /* distributed mat input */
Mat_MPIAIJ *mat = (Mat_MPIAIJ*)A->data;
aa=(Mat_SeqAIJ*)(mat->A)->data;
bb=(Mat_SeqAIJ*)(mat->B)->data;
ai=aa->i; aj=aa->j;
bi=bb->i; bj=bb->j;
#if defined(PETSC_USE_COMPLEX)
av=(doublecomplex*)aa->a;
bv=(doublecomplex*)bb->a;
#else
av=aa->a;
bv=bb->a;
#endif
rstart = A->rmap->rstart;
nz = aa->nz + bb->nz;
garray = mat->garray;
if (lu->options.Fact == DOFACT) { /* first numeric factorization */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zallocateA_dist",zallocateA_dist(m, nz, &lu->val, &lu->col, &lu->row));
#else
PetscStackCall("SuperLU_DIST:dallocateA_dist",dallocateA_dist(m, nz, &lu->val, &lu->col, &lu->row));
#endif
} else { /* successive numeric factorization, sparsity pattern and perm_c are reused. */
/* Destroy_CompRowLoc_Matrix_dist(&lu->A_sup); */ /* this leads to crash! However, see SuperLU_DIST_2.5/EXAMPLE/pzdrive2.c */
if (lu->FactPattern == SamePattern_SameRowPerm) {
lu->options.Fact = SamePattern_SameRowPerm; /* matrix has similar numerical values */
} else {
PetscStackCall("SuperLU_DIST:Destroy_LU",Destroy_LU(N, &lu->grid, &lu->LUstruct)); /* Deallocate storage associated with the L and U matrices. */
lu->options.Fact = SamePattern;
}
}
nz = 0;
for (i=0; i<m; i++) {
lu->row[i] = nz;
countA = ai[i+1] - ai[i];
countB = bi[i+1] - bi[i];
ajj = aj + ai[i]; /* ptr to the beginning of this row */
bjj = bj + bi[i];
/* B part, smaller col index */
colA_start = rstart + ajj[0]; /* the smallest global col index of A */
jB = 0;
for (j=0; j<countB; j++) {
jcol = garray[bjj[j]];
if (jcol > colA_start) {
jB = j;
break;
}
lu->col[nz] = jcol;
lu->val[nz++] = *bv++;
if (j==countB-1) jB = countB;
}
/* A part */
for (j=0; j<countA; j++) {
lu->col[nz] = rstart + ajj[j];
lu->val[nz++] = *av++;
}
/* B part, larger col index */
for (j=jB; j<countB; j++) {
lu->col[nz] = garray[bjj[j]];
lu->val[nz++] = *bv++;
}
}
lu->row[m] = nz;
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCreate_CompRowLoc_Matrix_dist",zCreate_CompRowLoc_Matrix_dist(&lu->A_sup, M, N, nz, m, rstart,lu->val, lu->col, lu->row, SLU_NR_loc, SLU_Z, SLU_GE));
#else
PetscStackCall("SuperLU_DIST:dCreate_CompRowLoc_Matrix_dist",dCreate_CompRowLoc_Matrix_dist(&lu->A_sup, M, N, nz, m, rstart,lu->val, lu->col, lu->row, SLU_NR_loc, SLU_D, SLU_GE));
#endif
}
/* Factor the matrix. */
PetscStackCall("SuperLU_DIST:PStatInit",PStatInit(&stat)); /* Initialize the statistics variables. */
if (lu->MatInputMode == GLOBAL) { /* global mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx_ABglobal",pzgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, M, 0,&lu->grid, &lu->LUstruct, berr, &stat, &sinfo));
#else
PetscStackCall("SuperLU_DIST:pdgssvx_ABglobal",pdgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, M, 0,&lu->grid, &lu->LUstruct, berr, &stat, &sinfo));
#endif
} else { /* distributed mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx",pzgssvx(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, m, 0, &lu->grid,&lu->LUstruct, &lu->SOLVEstruct, berr, &stat, &sinfo));
if (sinfo) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pzgssvx fails, info: %d\n",sinfo);
#else
PetscStackCall("SuperLU_DIST:pdgssvx",pdgssvx(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, m, 0, &lu->grid,&lu->LUstruct, &lu->SOLVEstruct, berr, &stat, &sinfo));
if (sinfo) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pdgssvx fails, info: %d\n",sinfo);
#endif
}
if (lu->MatInputMode == GLOBAL && size > 1) {
ierr = MatDestroy(&A_seq);CHKERRQ(ierr);
}
if (lu->options.PrintStat) {
PStatPrint(&lu->options, &stat, &lu->grid); /* Print the statistics. */
}
PStatFree(&stat);
if (size > 1) {
F_diag = ((Mat_MPIAIJ*)(F)->data)->A;
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
(F)->preallocated = PETSC_TRUE;
lu->options.Fact = FACTORED; /* The factored form of A is supplied. Local option used by this func. only */
PetscFunctionReturn(0);
}
static PetscErrorCode MatPartitioningApply_Party(MatPartitioning part, IS * partitioning)
{
PetscErrorCode ierr;
int *locals, *parttab = NULL, rank, size;
Mat mat = part->adj, matMPI, matSeq;
int nb_locals;
Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
MatPartitioning_Party *party = (MatPartitioning_Party *) part->data;
PetscTruth flg;
#ifdef PETSC_HAVE_UNISTD_H
int fd_stdout, fd_pipe[2], count,err;
#endif
PetscFunctionBegin;
/* check if the matrix is sequential, use MatGetSubMatrices if necessary */
ierr = PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)mat)->comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
if (size > 1) {
int M, N;
IS isrow, iscol;
Mat *A;
if (flg) SETERRQ(PETSC_ERR_SUP,"Distributed matrix format MPIAdj is not supported for sequential partitioners");
ierr = PetscPrintf(((PetscObject)part)->comm,"Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);
ierr = MatGetSize(mat, &M, &N);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);CHKERRQ(ierr);
ierr = ISDestroy(isrow);CHKERRQ(ierr);
ierr = ISDestroy(iscol);CHKERRQ(ierr);
matSeq = *A;
ierr = PetscFree(A);CHKERRQ(ierr);
} else {
matSeq = mat;
}
/* check for the input format that is supported only for a MPIADJ type
and set it to matMPI */
if (!flg) {
ierr = MatConvert(matSeq, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);CHKERRQ(ierr);
} else {
matMPI = matSeq;
}
adj = (Mat_MPIAdj *) matMPI->data; /* finaly adj contains adjacency graph */
{
/* Party library arguments definition */
int n = mat->rmap->N; /* number of vertices in full graph */
int *edge_p = adj->i; /* start of edge list for each vertex */
int *edge = adj->j; /* edge list data */
int *vertex_w = NULL; /* weights for all vertices */
int *edge_w = NULL; /* weights for all edges */
float *x = NULL, *y = NULL, *z = NULL; /* coordinates for inertial method */
int p = part->n; /* number of parts to create */
int *part_party; /* set number of each vtx (length n) */
int cutsize; /* number of edge cut */
char *global = party->global_method; /* global partitioning algorithm */
char *local = party->local_method; /* local partitioning algorithm */
int redl = party->nbvtxcoarsed; /* how many vertices to coarsen down to? */
char *redm = party->redm;
char *redo = party->redo;
int rec = party->rec;
int output = party->output;
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &part_party);CHKERRQ(ierr);
/* redirect output to buffer party->mesg_log */
#ifdef PETSC_HAVE_UNISTD_H
fd_stdout = dup(1);
pipe(fd_pipe);
close(1);
dup2(fd_pipe[1], 1);
ierr = PetscMalloc(SIZE_LOG * sizeof(char), &(party->mesg_log));CHKERRQ(ierr);
#endif
/* library call */
party_lib_times_start();
ierr = party_lib(n, vertex_w, x, y, z, edge_p, edge, edge_w,
p, part_party, &cutsize, redl, redm, redo,
global, local, rec, output);
party_lib_times_output(output);
part_info(n, vertex_w, edge_p, edge, edge_w, p, part_party, output);
#ifdef PETSC_HAVE_UNISTD_H
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on stdout");
count =
read(fd_pipe[0], party->mesg_log, (SIZE_LOG - 1) * sizeof(char));
if (count < 0)
count = 0;
party->mesg_log[count] = 0;
close(1);
dup2(fd_stdout, 1);
close(fd_stdout);
close(fd_pipe[0]);
close(fd_pipe[1]);
#endif
/* if in the call we got an error, we say it */
if (ierr) SETERRQ(PETSC_ERR_LIB, party->mesg_log);
parttab = part_party;
}
/* Creation of the index set */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)part)->comm, &size);CHKERRQ(ierr);
nb_locals = mat->rmap->N / size;
locals = parttab + rank * nb_locals;
if (rank < mat->rmap->N % size) {
nb_locals++;
locals += rank;
} else {
locals += mat->rmap->N % size;
}
ierr = ISCreateGeneral(((PetscObject)part)->comm, nb_locals, locals, partitioning);CHKERRQ(ierr);
/* destroying old objects */
ierr = PetscFree(parttab);CHKERRQ(ierr);
if (matSeq != mat) {
ierr = MatDestroy(matSeq);CHKERRQ(ierr);
}
if (matMPI != mat) {
ierr = MatDestroy(matMPI);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MatColoringCreateSmallestLastWeights(MatColoring mc,PetscReal *weights)
{
PetscInt *degrees,*degb,*llprev,*llnext;
PetscInt j,i,s,e,n,nin,ln,lm,degree,maxdegree=0,bidx,idx,dist,distance=mc->dist;
Mat lG,*lGs;
IS ris;
PetscErrorCode ierr;
PetscInt *seen;
const PetscInt *gidx;
PetscInt *idxbuf;
PetscInt *distbuf;
PetscInt ncols,nxt,prv,cur;
const PetscInt *cols;
PetscBool isSEQAIJ;
Mat_SeqAIJ *aij;
PetscInt *Gi,*Gj,*rperm;
Mat G = mc->mat;
PetscReal *lweights,r;
PetscRandom rand;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(G,&s,&e);CHKERRQ(ierr);
n=e-s;
ierr = ISCreateStride(PetscObjectComm((PetscObject)G),n,s,1,&ris);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(G,1,&ris,distance+1);CHKERRQ(ierr);
ierr = ISSort(ris);CHKERRQ(ierr);
ierr = MatGetSubMatrices(G,1,&ris,&ris,MAT_INITIAL_MATRIX,&lGs);CHKERRQ(ierr);
lG = lGs[0];
ierr = PetscObjectTypeCompare((PetscObject)lG,MATSEQAIJ,&isSEQAIJ);CHKERRQ(ierr);
if (!isSEQAIJ) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"Requires an MPI/SEQAIJ Matrix");
ierr = MatGetSize(lG,&ln,&lm);CHKERRQ(ierr);
aij = (Mat_SeqAIJ*)lG->data;
Gi = aij->i;
Gj = aij->j;
ierr = PetscMalloc3(lm,&seen,lm,&idxbuf,lm,&distbuf);CHKERRQ(ierr);
ierr = PetscMalloc1(lm,°rees);CHKERRQ(ierr);
ierr = PetscMalloc1(lm,&lweights);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
seen[i]=-1;
lweights[i] = 1.;
}
ierr = ISGetIndices(ris,&gidx);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
bidx=-1;
ncols = Gi[i+1]-Gi[i];
cols = &(Gj[Gi[i]]);
degree = 0;
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
degree++;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
degrees[i] = degree;
if (degree > maxdegree) maxdegree = degree;
}
/* bucket by degree by some random permutation */
ierr = PetscRandomCreate(PetscObjectComm((PetscObject)mc),&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = PetscMalloc1(ln,&rperm);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
ierr = PetscRandomGetValueReal(rand,&r);CHKERRQ(ierr);
lweights[i] = r;
rperm[i]=i;
}
ierr = PetscSortRealWithPermutation(lm,lweights,rperm);CHKERRQ(ierr);
ierr = PetscMalloc1(maxdegree+1,°b);CHKERRQ(ierr);
ierr = PetscMalloc2(ln,&llnext,ln,&llprev);CHKERRQ(ierr);
for (i=0;i<maxdegree+1;i++) {
degb[i] = -1;
}
for (i=0;i<ln;i++) {
llnext[i] = -1;
llprev[i] = -1;
seen[i] = -1;
}
for (i=0;i<ln;i++) {
idx = rperm[i];
llnext[idx] = degb[degrees[idx]];
if (degb[degrees[idx]] > 0) llprev[degb[degrees[idx]]] = idx;
degb[degrees[idx]] = idx;
}
ierr = PetscFree(rperm);CHKERRQ(ierr);
/* remove the lowest degree one */
i=0;
nin=0;
while (i != maxdegree+1) {
for (i=1;i<maxdegree+1; i++) {
if (degb[i] > 0) {
cur = degb[i];
nin++;
degrees[cur] = 0;
degb[i] = llnext[cur];
bidx=-1;
ncols = Gi[cur+1]-Gi[cur];
cols = &(Gj[Gi[cur]]);
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
if (cols[j] != cur) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
nxt=llnext[idx];
prv=llprev[idx];
if (degrees[idx] > 0) {
/* change up the degree of the neighbors still in the graph */
if (lweights[idx] <= lweights[cur]) lweights[idx] = lweights[cur]+1;
if (nxt > 0) {
llprev[nxt] = prv;
}
if (prv > 0) {
llnext[prv] = nxt;
} else {
degb[degrees[idx]] = nxt;
}
degrees[idx]--;
llnext[idx] = degb[degrees[idx]];
llprev[idx] = -1;
if (degb[degrees[idx]] >= 0) {
llprev[degb[degrees[idx]]] = idx;
}
degb[degrees[idx]] = idx;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
}
break;
}
}
}
for (i=0;i<lm;i++) {
if (gidx[i] >= s && gidx[i] < e) {
weights[gidx[i]-s] = lweights[i];
}
}
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFree(degb);CHKERRQ(ierr);
ierr = PetscFree2(llnext,llprev);CHKERRQ(ierr);
ierr = PetscFree(degrees);CHKERRQ(ierr);
ierr = PetscFree(lweights);CHKERRQ(ierr);
ierr = ISRestoreIndices(ris,&gidx);CHKERRQ(ierr);
ierr = ISDestroy(&ris);CHKERRQ(ierr);
ierr = PetscFree3(seen,idxbuf,distbuf);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lGs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
