void PETSc::Create(
MPI_Comm Comm,
int ilower,
int iupper,
int d_nz,
int o_nz)
{
int n = iupper - ilower +1;
comm = Comm;
iLower = ilower;
iUpper = iupper;
MatCreateAIJ(PETSC_COMM_WORLD,n,n,PETSC_DECIDE,PETSC_DECIDE,
d_nz,PETSC_NULL,o_nz,PETSC_NULL,&A);
ierr = PetscObjectSetName((PetscObject) A, "A");
ierr = MatSetFromOptions(A);
// b
ierr = VecCreate(PETSC_COMM_WORLD, &b);
ierr = PetscObjectSetName((PetscObject) b, "b");
ierr = VecSetSizes(b, n, PETSC_DECIDE);
ierr = VecSetFromOptions(b);
ierr = VecCreate(PETSC_COMM_WORLD,&x);
ierr = PetscObjectSetName((PetscObject) x, "X");
ierr = VecSetSizes(x, n, PETSC_DECIDE);
ierr = VecSetFromOptions(x);
}
int main(int argc, char **argv){
SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
int N[3] = {20, 25, 1}, Npml[3] = {5, 7, 0}, Nc = 2,
b[3][2] = {{-1, 0}, {1, 0}, {0, 0}}, LowerPML = 0, BCPeriod = 3;
double h[3] = {0.2, 0.3, 0.1};
int i, Nxyzr = 2*N[0]*N[1]*N[2];
double sigma[3], *muinv;
Vec muinvpml;
muinv = (double *) malloc(sizeof(double)*3*Nxyzr );
for(i=0; i<3; i++) sigma[i] = pmlsigma(1e-25, Npml[i]*h[i]);
MuinvPMLFull(PETSC_COMM_SELF, &muinvpml, N[0], N[1], N[2], Npml[0],
Npml[1], Npml[2], sigma[0], sigma[1], sigma[2], 1.0, LowerPML);
AddMuAbsorption(muinv, muinvpml, 1e25, 0);
VecDestroy(&muinvpml);
Mat Moperator;
MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, PETSC_DECIDE, Nc*Nxyzr,
Nc*Nxyzr, 26, NULL, 26, NULL, &Moperator);
MoperatorGeneralFill(Moperator, N[0], N[1], N[2], h[0], h[1], h[2],
b[0], b[1], b[2], muinv, BCPeriod, 0, Nc);
Assemble(Moperator);
PetscObjectSetName((PetscObject) Moperator, "M");
OutputMat(PETSC_COMM_WORLD, Moperator, filenameComm, "M.m");
SlepcFinalize();
return 0;
}
PETSC_EXTERN void PETSC_STDCALL matcreateaij_(MPI_Comm *comm,PetscInt *m,PetscInt *n,PetscInt *M,PetscInt *N,PetscInt *d_nz,PetscInt *d_nnz,PetscInt *o_nz,PetscInt *o_nnz,Mat *newmat,PetscErrorCode *ierr)
{
CHKFORTRANNULLINTEGER(d_nnz);
CHKFORTRANNULLINTEGER(o_nnz);
*ierr = MatCreateAIJ(MPI_Comm_f2c(*(MPI_Fint*)&*comm),*m,*n,*M,*N,*d_nz,d_nnz,*o_nz,o_nnz,newmat);
}
int main(int argc,char **argv)
{
Mat A,B;
PetscInt m = 7,n,i,rstart,rend,cols[3];
PetscErrorCode ierr;
PetscScalar v[3];
PetscBool equal;
const char *eq[2];
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
n = m;
/* ------- Assemble matrix, --------- */
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,m,n,0,0,0,0,&A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
if (!rstart) {
cols[0] = 0;
cols[1] = 1;
v[0] = 2.0; v[1] = -1.0;
ierr = MatSetValues(A,1,&rstart,2,cols,v,INSERT_VALUES);CHKERRQ(ierr);
rstart++;
}
if (rend == m) {
rend--;
cols[0] = rend-1;
cols[1] = rend;
v[0] = -1.0; v[1] = 2.0;
ierr = MatSetValues(A,1,&rend,2,cols,v,INSERT_VALUES);CHKERRQ(ierr);
}
v[0] = -1.0; v[1] = 2.0; v[2] = -1.0;
for (i=rstart; i<rend; i++) {
cols[0] = i-1;
cols[1] = i;
cols[2] = i+1;
ierr = MatSetValues(A,1,&i,3,cols,v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr);
ierr = MatEqual(A,B,&equal);
eq[0] = "not equal";
eq[1] = "equal";
ierr = PetscPrintf(PETSC_COMM_WORLD,"Matrices are %s\n",eq[equal]);CHKERRQ(ierr);
/* Free data structures */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat C;
PetscInt i,j,m = 3,n = 2,Ii,J,col = 0;
PetscMPIInt size,rank;
PetscErrorCode ierr;
PetscScalar v;
Vec yy;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-col",&col,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
n = 2*size;
/* create the matrix for the five point stencil, YET AGAIN*/
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,
m*n,m*n,5,NULL,5,NULL,&C);CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=2*rank; j<2*rank+2; j++) {
v = -1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&yy);CHKERRQ(ierr);
ierr = VecSetSizes(yy,PETSC_DECIDE,m*n);CHKERRQ(ierr);
ierr = VecSetFromOptions(yy);CHKERRQ(ierr);
ierr = MatGetColumnVector(C,yy,col);CHKERRQ(ierr);
ierr = VecView(yy,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&yy);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat C,A;
PetscScalar v;
PetscInt i,j,m = 4,n = 4,Ii,J,Istart,Iend;
PetscMPIInt rank,size;
PetscErrorCode ierr;
PetscBool equal=PETSC_FALSE;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
n = m;
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,
m*n,m*n,5,PETSC_NULL,5,PETSC_NULL,&C);CHKERRQ(ierr);
/* create the symmetric matrix for the five point stencil */
ierr = MatGetOwnershipRange(C,&Istart,&Iend);CHKERRQ(ierr);
for (Ii=Istart; Ii<Iend; Ii++) {
v = -1.0; i = Ii/n; j = Ii - i*n;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatTranspose(C,MAT_INITIAL_MATRIX,&A);CHKERRQ(ierr);
ierr = MatEqual(C,A,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"C != C^T");
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
Mat A;
MPI_Comm comm;
PetscInt n=5,m=5,*dnnz,*onnz,i,rstart,rend,M,N;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,0,help);if (ierr) return ierr;
comm = MPI_COMM_WORLD;
ierr = PetscMalloc2(m,&dnnz,m,&onnz);CHKERRQ(ierr);
for (i=0; i<m; i++) {
dnnz[i] = 1;
onnz[i] = 1;
}
ierr = MatCreateAIJ(comm,m,n,PETSC_DETERMINE,PETSC_DETERMINE,PETSC_DECIDE,dnnz,PETSC_DECIDE,onnz,&A);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = PetscFree2(dnnz,onnz);CHKERRQ(ierr);
/* This assembly shrinks memory because we do not insert enough number of values */
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* MatResetPreallocation restores the memory required by users */
ierr = MatResetPreallocation(A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatSetValue(A,i,i,2.0,INSERT_VALUES);CHKERRQ(ierr);
if (rend<N) {
ierr = MatSetValue(A,i,rend,1.0,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode PCGAMGCreateGraph(const Mat Amat, Mat *a_Gmat)
{
PetscErrorCode ierr;
PetscInt Istart,Iend,Ii,jj,kk,ncols,nloc,NN,MM,bs;
PetscMPIInt rank, size;
MPI_Comm comm;
Mat Gmat;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Amat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Amat, &Istart, &Iend);CHKERRQ(ierr);
ierr = MatGetSize(Amat, &MM, &NN);CHKERRQ(ierr);
ierr = MatGetBlockSize(Amat, &bs);CHKERRQ(ierr);
nloc = (Iend-Istart)/bs;
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
if (bs > 1) {
const PetscScalar *vals;
const PetscInt *idx;
PetscInt *d_nnz, *o_nnz;
/* count nnz, there is sparcity in here so this might not be enough */
ierr = PetscMalloc1(nloc, &d_nnz);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc, &o_nnz);CHKERRQ(ierr);
for (Ii = Istart, jj = 0; Ii < Iend; Ii += bs, jj++) {
d_nnz[jj] = 0;
for (kk=0; kk<bs; kk++) {
ierr = MatGetRow(Amat,Ii+kk,&ncols,0,0);CHKERRQ(ierr);
if (ncols > d_nnz[jj]) {
d_nnz[jj] = ncols; /* very pessimistic but could be too low in theory */
o_nnz[jj] = ncols;
if (d_nnz[jj] > nloc) d_nnz[jj] = nloc;
if (o_nnz[jj] > (NN/bs-nloc)) o_nnz[jj] = NN/bs-nloc;
}
ierr = MatRestoreRow(Amat,Ii+kk,&ncols,0,0);CHKERRQ(ierr);
}
}
/* get scalar copy (norms) of matrix -- AIJ specific!!! */
ierr = MatCreateAIJ(comm, nloc, nloc, PETSC_DETERMINE, PETSC_DETERMINE,0, d_nnz, 0, o_nnz, &Gmat);CHKERRQ(ierr);
ierr = PetscFree(d_nnz);CHKERRQ(ierr);
ierr = PetscFree(o_nnz);CHKERRQ(ierr);
for (Ii = Istart; Ii < Iend; Ii++) {
PetscInt dest_row = Ii/bs;
ierr = MatGetRow(Amat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
for (jj=0; jj<ncols; jj++) {
PetscInt dest_col = idx[jj]/bs;
PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
ierr = MatSetValues(Gmat,1,&dest_row,1,&dest_col,&sv,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatRestoreRow(Amat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(Gmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Gmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
} else {
/* just copy scalar matrix - abs() not taken here but scaled later */
ierr = MatDuplicate(Amat, MAT_COPY_VALUES, &Gmat);CHKERRQ(ierr);
}
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
*a_Gmat = Gmat;
PetscFunctionReturn(0);
}
PetscErrorCode BearQueryMat(PetscInt s, PetscScalar c, Mat invL1, Mat invU1, Mat invL2, Mat invU2, Mat H12, Mat H21, Vec order){
PetscErrorCode err;
PetscInt n1, n2, n, M, N;
PetscInt oseed;
PetscScalar val, one = 1.0;
PetscMPIInt size;
PetscLogDouble tic, toc;
Mat r = NULL;
Mat r1 = NULL, q1 = NULL, t1_1 = NULL, t1_2 = NULL, t1_3 = NULL, t1_4 = NULL, t1_5 = NULL; // dimension: n1
Mat r2 = NULL, q2 = NULL, q_tilda = NULL, t2_1 = NULL, t2_2 = NULL, t2_3 = NULL; // dimension: n2_idx
Vec vr=NULL, vr1=NULL, vr2=NULL;
PetscInt col = 0;
err = MPI_Comm_size(PETSC_COMM_WORLD, &size); CHKERRQ(err);
err = MatGetSize(H12, &n1, &n2); CHKERRQ(err);
n = n1 + n2;
err = PetscPrintf(PETSC_COMM_WORLD, "n1: %d, n2: %d\n", n1, n2); CHKERRQ(err);
err = MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, 1, n, size, 1, NULL, 1, NULL, &r); CHKERRQ(err);
err = MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, 1, n1, size, 1, NULL, 1, NULL, &q1); CHKERRQ(err);
err = MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, 1, n2, size, 1, NULL, 1, NULL, &q2); CHKERRQ(err);
// err = MatCreate(PETSC_COMM_WORLD, &q2); CHKERRQ(err);
// err = MatSetSizes(q2, PETSC_DECIDE, PETSC_DECIDE, n2, 1); CHKERRQ(err);
// err = MatSetType(q2, MATAIJ); CHKERRQ(err);
// err = MatSetUp(q2);
s = s - 1; // shift -1 for zero-based index
err = VecGetValues(order, 1, &s, &val); CHKERRQ(err);
oseed = (PetscInt) val;
// err = PetscPrintf(PETSC_COMM_WORLD, "Given seed: %d, Reorered seed: %d (0 ~ n-1)\n", s, oseed); CHKERRQ(err);
if(oseed < n1){
//err = MatSetValues(q1, 1, &oseed, 1, &col, &one, INSERT_VALUES); CHKERRQ(err);
err = MatSetValue(q1, oseed, col, one, INSERT_VALUES); CHKERRQ(err);
}else{
oseed = oseed - n1;
//err = MatSetValues(q2, 1, &oseed, 1, &col, &one, INSERT_VALUES); CHKERRQ(err);
err = MatSetValue(q2, oseed, col, one, INSERT_VALUES); CHKERRQ(err);
//err = printVecSum(q2);
}
err = MatAssemblyBegin(q1, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = MatAssemblyEnd(q1, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = MatAssemblyBegin(q2, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = MatAssemblyEnd(q2, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = printMatInfo("q1", q1);
err = printMatInfo("q2", q2);
//err = MatView(q1, PETSC_VIEWER_STDOUT_WORLD);
//err = MatView(q2, PETSC_VIEWER_STDOUT_WORLD);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &r1); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_1); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_2); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_3); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_4); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_5); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &r2); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &q_tilda); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &t2_1); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &t2_2); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &t2_3); CHKERRQ(err);
err = MatMatMult(invL1, q1, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_1); CHKERRQ(err);
err = MatMatMult(invU1, t1_1, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_2); CHKERRQ(err);
err = MatMatMult(H21, t1_2, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t2_1); CHKERRQ(err);
err = MatScale(t2_1, -1.0); CHKERRQ(err);
err = MatAXPY(t2_1, 1.0, q2, DIFFERENT_NONZERO_PATTERN); CHKERRQ(err);
//MatView(t1_1, PETSC_VIEWER_STDOUT_WORLD);
err = PetscTime(&tic);
err = MatMatMult(invL2, t2_1, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t2_2); CHKERRQ(err);
err = MatMatMult(invU2, t2_2, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &r2); CHKERRQ(err);
err = PetscTime(&toc);
err = PetscPrintf(PETSC_COMM_WORLD, "running time: %f sec\n", toc-tic); CHKERRQ(err);
err = MatMatMult(H12, r2, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_3); CHKERRQ(err);
err = MatScale(t1_3, -1.0); CHKERRQ(err);
err = MatAXPY(t1_3, 1.0, q1, DIFFERENT_NONZERO_PATTERN); CHKERRQ(err);
err = MatMatMult(invL1, t1_3, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_5); CHKERRQ(err);
err = MatMatMult(invU1, t1_5, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &r1); CHKERRQ(err);
//MatView(r1, PETSC_VIEWER_STDOUT_WORLD);
MatGetSize(r1, &M, &N);
PetscPrintf(PETSC_COMM_WORLD, "%d %d\n", M, N);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n1, &vr1);
err = MatGetColumnVector(r1, vr1, 0);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n2, &vr2);
err = MatGetColumnVector(r2, vr2, 0);
err = printMatInfo("r2", r2);
/*
// Start matrix-vec multiplications
err = MatMult(invU2, t2_2, r2); CHKERRQ(err);
err = MatMult(H12, r2, t1_3); CHKERRQ(err);
err = VecAXPBYPCZ(t1_4, 1.0, -1.0, 0.0, q1, t1_3); CHKERRQ(err);
err = MatMult(invL1, t1_4, t1_5); CHKERRQ(err);
err = MatMult(invU1, t1_5, r1); CHKERRQ(err);
//err = printVecSum(r1);
//err = VecView(r2, PETSC_VIEWER_STDOUT_WORLD);
// Concatenate r1 and r2
err = VecMerge(r1, r2, r); CHKERRQ(err);
err = VecScale(r, c); CHKERRQ(err);
//err = VecView(r, PETSC_VIEWER_STDOUT_WORLD);
//err = VecDuplicate(r, &or); CHKERRQ(err);
err = VecReorder(r, order, or); CHKERRQ(err);
//err = VecView(or, PETSC_VIEWER_STDOUT_WORLD);
*/
err = MatDestroy(&r); CHKERRQ(err);
err = MatDestroy(&r1); CHKERRQ(err);
err = MatDestroy(&q1); CHKERRQ(err);
err = MatDestroy(&t1_1); CHKERRQ(err);
err = MatDestroy(&t1_2); CHKERRQ(err);
err = MatDestroy(&t1_3); CHKERRQ(err);
err = MatDestroy(&t1_4); CHKERRQ(err);
err = MatDestroy(&t1_5); CHKERRQ(err);
err = MatDestroy(&r2); CHKERRQ(err);
err = MatDestroy(&q2); CHKERRQ(err);
err = MatDestroy(&q_tilda); CHKERRQ(err);
err = MatDestroy(&t2_1); CHKERRQ(err);
err = MatDestroy(&t2_2); CHKERRQ(err);
err = MatDestroy(&t2_3); CHKERRQ(err);
return err;
}
PetscErrorCode PCGAMGFilterGraph(Mat *a_Gmat,const PetscReal vfilter,const PetscBool symm,const PetscInt verbose)
{
PetscErrorCode ierr;
PetscInt Istart,Iend,Ii,jj,ncols,nnz0,nnz1, NN, MM, nloc;
PetscMPIInt rank, size;
Mat Gmat = *a_Gmat, tGmat, matTrans;
MPI_Comm comm;
const PetscScalar *vals;
const PetscInt *idx;
PetscInt *d_nnz, *o_nnz;
Vec diag;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Gmat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Gmat, &Istart, &Iend);CHKERRQ(ierr);
nloc = Iend - Istart;
ierr = MatGetSize(Gmat, &MM, &NN);CHKERRQ(ierr);
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
/* scale Gmat so filter works */
ierr = MatGetVecs(Gmat, &diag, 0);CHKERRQ(ierr);
ierr = MatGetDiagonal(Gmat, diag);CHKERRQ(ierr);
ierr = VecReciprocal(diag);CHKERRQ(ierr);
ierr = VecSqrtAbs(diag);CHKERRQ(ierr);
ierr = MatDiagonalScale(Gmat, diag, diag);CHKERRQ(ierr);
ierr = VecDestroy(&diag);CHKERRQ(ierr);
if (symm) {
ierr = MatTranspose(Gmat, MAT_INITIAL_MATRIX, &matTrans);CHKERRQ(ierr);
}
/* filter - dup zeros out matrix */
ierr = PetscMalloc1(nloc, &d_nnz);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc, &o_nnz);CHKERRQ(ierr);
for (Ii = Istart, jj = 0; Ii < Iend; Ii++, jj++) {
ierr = MatGetRow(Gmat,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
d_nnz[jj] = ncols;
o_nnz[jj] = ncols;
ierr = MatRestoreRow(Gmat,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
if (symm) {
ierr = MatGetRow(matTrans,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
d_nnz[jj] += ncols;
o_nnz[jj] += ncols;
ierr = MatRestoreRow(matTrans,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
}
if (d_nnz[jj] > nloc) d_nnz[jj] = nloc;
if (o_nnz[jj] > (MM-nloc)) o_nnz[jj] = MM - nloc;
}
ierr = MatCreateAIJ(comm, nloc, nloc, MM, MM, 0, d_nnz, 0, o_nnz, &tGmat);CHKERRQ(ierr);
ierr = PetscFree(d_nnz);CHKERRQ(ierr);
ierr = PetscFree(o_nnz);CHKERRQ(ierr);
if (symm) {
ierr = MatDestroy(&matTrans);CHKERRQ(ierr);
}
for (Ii = Istart, nnz0 = nnz1 = 0; Ii < Iend; Ii++) {
ierr = MatGetRow(Gmat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
for (jj=0; jj<ncols; jj++,nnz0++) {
PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
if (PetscRealPart(sv) > vfilter) {
nnz1++;
if (symm) {
sv *= 0.5;
ierr = MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(tGmat,1,&idx[jj],1,&Ii,&sv,ADD_VALUES);CHKERRQ(ierr);
} else {
ierr = MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatRestoreRow(Gmat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(tGmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(tGmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
if (verbose) {
if (verbose == 1) {
ierr = PetscPrintf(comm,"\t[%d]%s %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%d)\n",rank,__FUNCT__,
100.*(double)nnz1/(double)nnz0,vfilter,(double)nnz0/(double)nloc,MM);CHKERRQ(ierr);
} else {
PetscInt nnz[2],out[2];
nnz[0] = nnz0; nnz[1] = nnz1;
ierr = MPI_Allreduce(nnz, out, 2, MPIU_INT, MPI_SUM, comm);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"\t[%d]%s %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%d)\n",rank,__FUNCT__,
100.*(double)out[1]/(double)out[0],vfilter,(double)out[0]/(double)MM,MM);CHKERRQ(ierr);
}
}
ierr = MatDestroy(&Gmat);CHKERRQ(ierr);
*a_Gmat = tGmat;
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
IS is0a,is0b,is0,is1,isl0a,isl0b,isl0,isl1;
Mat A,Aexplicit;
PetscBool usenest;
PetscMPIInt rank,size;
PetscInt i,j;
PetscInitialize(&argc,&argv,PETSC_NULL,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
{
const PetscInt ix0a[] = {rank*2+0},ix0b[] = {rank*2+1},ix0[] = {rank*3+0,rank*3+1},ix1[] = {rank*3+2};
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,ix0a,PETSC_COPY_VALUES,&is0a);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,ix0b,PETSC_COPY_VALUES,&is0b);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,2,ix0,PETSC_COPY_VALUES,&is0);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,ix1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
}
{
ierr = ISCreateStride(PETSC_COMM_SELF,6,0,1,&isl0);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,3,0,1,&isl0a);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,3,3,1,&isl0b);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,3,6,1,&isl1);CHKERRQ(ierr);
}
usenest = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-nest",&usenest,PETSC_NULL);CHKERRQ(ierr);
if (usenest) {
ISLocalToGlobalMapping l2g;
const PetscInt l2gind[3] = {(rank-1+size)%size,rank,(rank+1)%size};
Mat B[9];
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,3,l2gind,PETSC_COPY_VALUES,&l2g);CHKERRQ(ierr);
for (i=0; i<9; i++) {
ierr = MatCreateAIJ(PETSC_COMM_WORLD,1,1,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,PETSC_NULL,PETSC_DECIDE,PETSC_NULL,&B[i]);CHKERRQ(ierr);
ierr = MatSetUp(B[i]);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(B[i],l2g,l2g);CHKERRQ(ierr);
}
{
const IS isx[] = {is0a,is0b};
const Mat Bx00[] = {B[0],B[1],B[3],B[4]},Bx01[] = {B[2],B[5]},Bx10[] = {B[6],B[7]};
Mat B00,B01,B10;
ierr = MatCreateNest(PETSC_COMM_WORLD,2,isx,2,isx,Bx00,&B00);CHKERRQ(ierr);
ierr = MatSetUp(B00);CHKERRQ(ierr);
ierr = MatCreateNest(PETSC_COMM_WORLD,2,isx,1,PETSC_NULL,Bx01,&B01);CHKERRQ(ierr);
ierr = MatSetUp(B01);CHKERRQ(ierr);
ierr = MatCreateNest(PETSC_COMM_WORLD,1,PETSC_NULL,2,isx,Bx10,&B10);CHKERRQ(ierr);
ierr = MatSetUp(B10);CHKERRQ(ierr);
{
Mat By[] = {B00,B01,B10,B[8]};
IS isy[] = {is0,is1};
ierr = MatCreateNest(PETSC_COMM_WORLD,2,isy,2,isy,By,&A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
}
ierr = MatDestroy(&B00);CHKERRQ(ierr);
ierr = MatDestroy(&B01);CHKERRQ(ierr);
ierr = MatDestroy(&B10);CHKERRQ(ierr);
}
for (i=0; i<9; i++) {ierr = MatDestroy(&B[i]);CHKERRQ(ierr);}
ierr = ISLocalToGlobalMappingDestroy(&l2g);CHKERRQ(ierr);
} else {
ISLocalToGlobalMapping l2g;
PetscInt l2gind[9];
for (i=0; i<3; i++) for (j=0; j<3; j++) l2gind[3*i+j] = ((rank-1+j+size) % size)*3 + i;
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,9,l2gind,PETSC_COPY_VALUES,&l2g);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,3,3,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,PETSC_NULL,PETSC_DECIDE,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(A,l2g,l2g);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&l2g);CHKERRQ(ierr);
}
{
Mat A00,A11,A0a0a,A0a0b;
ierr = MatGetLocalSubMatrix(A,isl0,isl0,&A00);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(A,isl1,isl1,&A11);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(A00,isl0a,isl0a,&A0a0a);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(A00,isl0a,isl0b,&A0a0b);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0a,0,0,100*rank+1,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0a,0,1,100*rank+2,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0a,2,2,100*rank+9,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0b,1,1,100*rank+50+5,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A11,0,0,1000*(rank+1)+1,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A11,1,2,1000*(rank+1)+6,ADD_VALUES);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A00,isl0a,isl0a,&A0a0a);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A00,isl0a,isl0b,&A0a0b);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A,isl0,isl0,&A00);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A,isl1,isl1,&A11);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatComputeExplicitOperator(A,&Aexplicit);CHKERRQ(ierr);
ierr = MatView(Aexplicit,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&Aexplicit);CHKERRQ(ierr);
ierr = ISDestroy(&is0a);CHKERRQ(ierr);
ierr = ISDestroy(&is0b);CHKERRQ(ierr);
ierr = ISDestroy(&is0);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&isl0a);CHKERRQ(ierr);
ierr = ISDestroy(&isl0b);CHKERRQ(ierr);
ierr = ISDestroy(&isl0);CHKERRQ(ierr);
ierr = ISDestroy(&isl1);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
/*
Forms the interpolation (and restriction) operator from
coarse grid to fine.
*/
PetscErrorCode FormInterpolation(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i,j,i_start,m_fine,j_start,m,n,*idx;
PetscInt m_ghost,n_ghost,*idx_c,m_ghost_c,n_ghost_c,m_coarse;
PetscInt row,i_start_ghost,j_start_ghost,cols[4], m_c;
PetscInt nc,ratio = user->ratio,m_c_local,m_fine_local;
PetscInt i_c,j_c,i_start_c,j_start_c,n_c,i_start_ghost_c,j_start_ghost_c,col;
PetscScalar v[4],x,y, one = 1.0;
Mat mat;
Vec Rscale;
ierr = DMDAGetCorners(user->fine.da,&i_start,&j_start,0,&m,&n,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->fine.da,&i_start_ghost,&j_start_ghost,0,&m_ghost,&n_ghost,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(user->fine.da,PETSC_NULL,&idx);CHKERRQ(ierr);
ierr = DMDAGetCorners(user->coarse.da,&i_start_c,&j_start_c,0,&m_c,&n_c,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->coarse.da,&i_start_ghost_c,&j_start_ghost_c,0,&m_ghost_c,&n_ghost_c,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(user->coarse.da,PETSC_NULL,&idx_c);CHKERRQ(ierr);
/* create interpolation matrix */
ierr = VecGetLocalSize(user->fine.x,&m_fine_local);CHKERRQ(ierr);
ierr = VecGetLocalSize(user->coarse.x,&m_c_local);CHKERRQ(ierr);
ierr = VecGetSize(user->fine.x,&m_fine);CHKERRQ(ierr);
ierr = VecGetSize(user->coarse.x,&m_coarse);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,m_fine_local,m_c_local,m_fine,m_coarse,
5,0,3,0,&mat);CHKERRQ(ierr);
/* loop over local fine grid nodes setting interpolation for those*/
for ( j=j_start; j<j_start+n; j++ ) {
for ( i=i_start; i<i_start+m; i++ ) {
/* convert to local "natural" numbering and then to PETSc global numbering */
row = idx[m_ghost*(j-j_start_ghost) + (i-i_start_ghost)];
i_c = (i/ratio); /* coarse grid node to left of fine grid node */
j_c = (j/ratio); /* coarse grid node below fine grid node */
/*
Only include those interpolation points that are truly
nonzero. Note this is very important for final grid lines
in x and y directions; since they have no right/top neighbors
*/
x = ((double)(i - i_c*ratio))/((double)ratio);
y = ((double)(j - j_c*ratio))/((double)ratio);
nc = 0;
/* one left and below; or we are right on it */
if (j_c < j_start_ghost_c || j_c > j_start_ghost_c+n_ghost_c) {
SETERRQ3(PETSC_COMM_SELF,1,"Sorry j %D %D %D",j_c,j_start_ghost_c,j_start_ghost_c+n_ghost_c);
}
if (i_c < i_start_ghost_c || i_c > i_start_ghost_c+m_ghost_c) {
SETERRQ3(PETSC_COMM_SELF,1,"Sorry i %D %D %D",i_c,i_start_ghost_c,i_start_ghost_c+m_ghost_c);
}
col = m_ghost_c*(j_c-j_start_ghost_c) + (i_c-i_start_ghost_c);
cols[nc] = idx_c[col];
v[nc++] = x*y - x - y + 1.0;
/* one right and below */
if (i_c*ratio != i) {
cols[nc] = idx_c[col+1];
v[nc++] = -x*y + x;
}
/* one left and above */
if (j_c*ratio != j) {
cols[nc] = idx_c[col+m_ghost_c];
v[nc++] = -x*y + y;
}
/* one right and above */
if (j_c*ratio != j && i_c*ratio != i) {
cols[nc] = idx_c[col+m_ghost_c+1];
v[nc++] = x*y;
}
ierr = MatSetValues(mat,1,&row,nc,cols,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecDuplicate(user->coarse.x,&Rscale);CHKERRQ(ierr);
ierr = VecSet(user->fine.x,one);CHKERRQ(ierr);
ierr = MatMultTranspose(mat,user->fine.x,Rscale);CHKERRQ(ierr);
ierr = VecReciprocal(Rscale);CHKERRQ(ierr);
user->Rscale = Rscale;
user->R = mat;
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt its,n,Nx=PETSC_DECIDE,Ny=PETSC_DECIDE,nlocal;
PetscMPIInt size;
PetscScalar one = 1.0;
PetscInt mx,my;
Mat A;
GridCtx fine_ctx;
KSP ksp;
PetscBool flg;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
/* set up discretization matrix for fine grid */
fine_ctx.mx = 9; fine_ctx.my = 9;
ierr = PetscOptionsGetInt(NULL,NULL,"-mx",&mx,&flg);CHKERRQ(ierr);
if (flg) fine_ctx.mx = mx;
ierr = PetscOptionsGetInt(NULL,NULL,"-my",&my,&flg);CHKERRQ(ierr);
if (flg) fine_ctx.my = my;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Fine grid size %D by %D\n",fine_ctx.mx,fine_ctx.my);CHKERRQ(ierr);
n = fine_ctx.mx*fine_ctx.my;
MPI_Comm_size(PETSC_COMM_WORLD,&size);
ierr = PetscOptionsGetInt(NULL,NULL,"-Nx",&Nx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Ny",&Ny,NULL);CHKERRQ(ierr);
/* Set up distributed array for fine grid */
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,fine_ctx.mx,
fine_ctx.my,Nx,Ny,1,1,NULL,NULL,&fine_ctx.da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(fine_ctx.da,&fine_ctx.x);CHKERRQ(ierr);
ierr = VecDuplicate(fine_ctx.x,&fine_ctx.b);CHKERRQ(ierr);
ierr = VecGetLocalSize(fine_ctx.x,&nlocal);CHKERRQ(ierr);
ierr = DMCreateLocalVector(fine_ctx.da,&fine_ctx.localX);CHKERRQ(ierr);
ierr = VecDuplicate(fine_ctx.localX,&fine_ctx.localF);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,nlocal,nlocal,n,n,5,NULL,3,NULL,&A);CHKERRQ(ierr);
ierr = FormJacobian_Grid(&fine_ctx,&A);CHKERRQ(ierr);
/* create linear solver */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/* set values for rhs vector */
ierr = VecSet(fine_ctx.b,one);CHKERRQ(ierr);
/* set options, then solve system */
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* calls PCSetFromOptions_ML if 'pc_type=ml' */
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPSolve(ksp,fine_ctx.b,fine_ctx.x);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %D\n",its);CHKERRQ(ierr);
/* free data structures */
ierr = VecDestroy(&fine_ctx.x);CHKERRQ(ierr);
ierr = VecDestroy(&fine_ctx.b);CHKERRQ(ierr);
ierr = DMDestroy(&fine_ctx.da);CHKERRQ(ierr);
ierr = VecDestroy(&fine_ctx.localX);CHKERRQ(ierr);
ierr = VecDestroy(&fine_ctx.localF);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int SolarEigenvaluesSolver(Mat M, Vec epsCurrent, Vec epspmlQ, Mat D)
{
PetscErrorCode ierr;
EPS eps;
PetscInt nconv;
Mat B;
int nrow, ncol;
ierr=MatGetSize(M,&nrow, &ncol); CHKERRQ(ierr);
ierr=MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, PETSC_DECIDE, nrow, ncol, 2, NULL, 2, NULL, &B); CHKERRQ(ierr);
ierr=PetscObjectSetName((PetscObject)B, "epsmatrix"); CHKERRQ(ierr);
if (D==PETSC_NULL)
{ // for purely real epsC, no absorption;
ierr=MatDiagonalSet(B,epsCurrent,INSERT_VALUES); CHKERRQ(ierr);
MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);
}
else
{
Vec epsC;
VecDuplicate(epsCurrent, &epsC);
ierr = VecPointwiseMult(epsC, epsCurrent,epspmlQ); CHKERRQ(ierr);
MatSetTwoDiagonals(B, epsC, D, 1.0);
VecDestroy(&epsC);
}
PetscPrintf(PETSC_COMM_WORLD,"!!!---computing eigenvalues---!!! \n");
ierr=EPSCreate(PETSC_COMM_WORLD, &eps); CHKERRQ(ierr);
ierr=EPSSetOperators(eps, M, B); CHKERRQ(ierr);
//ierr=EPSSetProblemType(eps,EPS_PGNHEP);CHKERRQ(ierr);
EPSSetFromOptions(eps);
PetscLogDouble t1, t2, tpast;
ierr = PetscTime(&t1);CHKERRQ(ierr);
ierr=EPSSolve(eps); CHKERRQ(ierr);
EPSGetConverged(eps, &nconv); CHKERRQ(ierr);
{
ierr = PetscTime(&t2);CHKERRQ(ierr);
tpast = t2 - t1;
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if(rank==0)
PetscPrintf(PETSC_COMM_SELF,"---The eigensolver time is %f s \n",tpast);
}
ierr = PetscPrintf(PETSC_COMM_WORLD,"Number of converged eigenpairs: %d\n\n",nconv);CHKERRQ(ierr);
double *krarray, *kiarray, *errorarray;
krarray = (double *) malloc(sizeof(double)*nconv);
kiarray = (double *) malloc(sizeof(double)*nconv);
errorarray =(double *) malloc(sizeof(double)*nconv);
int ni;
for(ni=0; ni<nconv; ni++)
{
ierr=EPSGetEigenpair(eps,ni, krarray+ni,kiarray+ni,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = EPSComputeRelativeError(eps,ni,errorarray+ni);CHKERRQ(ierr);
ierr=EPSComputeRelativeError(eps, ni, errorarray+ni );
}
PetscPrintf(PETSC_COMM_WORLD, "Now print the eigenvalues: \n");
for(ni=0; ni<nconv; ni++)
PetscPrintf(PETSC_COMM_WORLD," %g%+gi,", krarray[ni], kiarray[ni]);
PetscPrintf(PETSC_COMM_WORLD, "\n\nNow print the normalized eigenvalues: \n");
for(ni=0; ni<nconv; ni++)
PetscPrintf(PETSC_COMM_WORLD," %g%+gi,", sqrt(krarray[ni]+pow(omega,2))/(2*PI),kiarray[ni]);
PetscPrintf(PETSC_COMM_WORLD, "\n\nstart printing erros");
for(ni=0; ni<nconv; ni++)
PetscPrintf(PETSC_COMM_WORLD," %g,", errorarray[ni]);
PetscPrintf(PETSC_COMM_WORLD,"\n\n Finish EPS Solving !!! \n\n");
/*-- destroy vectors and free space --*/
EPSDestroy(&eps);
MatDestroy(&B);
free(krarray);
free(kiarray);
free(errorarray);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
TS ts; /* ODE integrator */
Vec U,V; /* solution will be stored here */
Vec F; /* residual vector */
Mat J; /* Jacobian matrix */
PetscMPIInt rank;
PetscScalar *u,*v;
AppCtx app;
PetscInt direction[2];
PetscBool terminate[2];
TSAdapt adapt;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,NULL,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
app.Cd = 0.0;
app.Cr = 0.9;
app.bounces = 0;
app.maxbounces = 10;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"ex44 options","");CHKERRQ(ierr);
ierr = PetscOptionsReal("-Cd","Drag coefficient","",app.Cd,&app.Cd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-Cr","Restitution coefficient","",app.Cr,&app.Cr,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-maxbounces","Maximum number of bounces","",app.maxbounces,&app.maxbounces,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
/*ierr = TSSetSaveTrajectory(ts);CHKERRQ(ierr);*/
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetType(ts,TSALPHA2);CHKERRQ(ierr);
ierr = TSSetDuration(ts,PETSC_MAX_INT,PETSC_MAX_REAL);CHKERRQ(ierr);
ierr = TSSetTimeStep(ts,0.1);CHKERRQ(ierr);
ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_STEPOVER);CHKERRQ(ierr);
ierr = TSGetAdapt(ts,&adapt);CHKERRQ(ierr);
ierr = TSAdaptSetStepLimits(adapt,0.0,0.5);CHKERRQ(ierr);
direction[0] = -1; terminate[0] = PETSC_FALSE;
direction[1] = -1; terminate[1] = PETSC_TRUE;
ierr = TSSetEventHandler(ts,2,direction,terminate,Event,PostEvent,&app);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,1,1,PETSC_DECIDE,PETSC_DECIDE,1,NULL,0,NULL,&J);CHKERRQ(ierr);
ierr = MatSetFromOptions(J);CHKERRQ(ierr);
ierr = MatSetUp(J);CHKERRQ(ierr);
ierr = MatCreateVecs(J,NULL,&F);CHKERRQ(ierr);
ierr = TSSetI2Function(ts,F,I2Function,&app);CHKERRQ(ierr);
ierr = TSSetI2Jacobian(ts,J,J,I2Jacobian,&app);CHKERRQ(ierr);
ierr = VecDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = TSGetI2Jacobian(ts,&J,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(J,&U,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(J,&V,NULL);CHKERRQ(ierr);
ierr = VecGetArray(U,&u);CHKERRQ(ierr);
ierr = VecGetArray(V,&v);CHKERRQ(ierr);
u[0] = 5.0*rank; v[0] = 20.0;
ierr = VecRestoreArray(U,&u);CHKERRQ(ierr);
ierr = VecRestoreArray(V,&v);CHKERRQ(ierr);
ierr = TS2SetSolution(ts,U,V);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = TSSolve(ts,NULL);CHKERRQ(ierr);
ierr = VecDestroy(&U);CHKERRQ(ierr);
ierr = VecDestroy(&V);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
* It's supposed the global matrix is assembled for all (free and dirichlet)
* nodes. For the current FVM used, where system of equation is assembled in
* a sub-domain by sub-domain approach, work becomes easier using all nodes.
* After that, it's desired to solve the system of equation only for
* the free ones as expected. Each rank needs to import some rows from the
* assembled matrix to a new matrix. The function below does this job.
*
* - nrows: number of rows will be imported and will be local to that rank.
* - rows: array with number of rows indices that correspond to a free node. *
* */
int MeshData::rowsToImport(pMesh theMesh, int &nrows, int *&rows){
Mat temp;
PetscErrorCode ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,numGF,numGF,0,PETSC_NULL,0,PETSC_NULL,&temp);
// get range of local owned rows of each process
const PetscInt *ranges[P_size()];
ierr = MatGetOwnershipRanges(temp,ranges);
//printf("[%d] - ranges[0] %d, ranges[1] %d, ranges[2] %d, ranges[3] %d\n",P_pid(),ranges[0][0],ranges[0][1],ranges[0][2],ranges[0][3]);
// each rank should take from matrix A row indices associated to free nodes
// that fill exactly the number of local rows of LHS, i.e,
// - rank 0 takes the first n1 rows from A, even if it is not local to rank 0.
// - rank 1 takes from the first n1 rows to n1+n2
// - rank 2 takes from the first n1+n2 rows to n1+n2+n3
// - (...)
// - rank p takes from the first sum(ni),i=1:p rows to sum(ni),i=i:p+1
int i,k;
int RANGE[P_size()];
for (i=0; i<P_size(); i++){
RANGE[i] = ranges[0][i+1]-ranges[0][i];
}
//printf("[%d] - RANGE %d %d %d\n",P_pid(),RANGE[0],RANGE[1],RANGE[2]);
const int from = ranges[0][P_pid()];
const int to = ranges[0][P_pid()+1];
//printf("[%d] - from %d to %d\n",P_pid(),from,to); exit(1);
ierr = MatDestroy(&temp); CHKERRQ(ierr);
// Inform which ROWS from A, on processor p, must be copied to assembly LHS matrix (free nodes). For each rank, the number of
// rows associated to free nodes cannot be forecast so a list is used.
list<int> freerowsList;
k = 0;
if (P_pid()==0){
for (i=1; i<=numGN; i++){
if ( !getDirichletValue(i,0) && k<to ){
freerowsList.push_back(i-1);
k++;
}
}
}
else{
for (i=1; i<=numGN; i++){
if ( !getDirichletValue(i,0) ){
if ( k>=from && k<to )
freerowsList.push_back(i-1);
k++;
}
}
}
// transfer row indices from list to array
nrows = freerowsList.size();
if ( !nrows ){
throw Exception(__LINE__,__FILE__,"numLocalRows = 0\n");
}
rows = new int[nrows];
i=0;
for(list<int>::iterator lit=freerowsList.begin(); lit!=freerowsList.end(); lit++){
rows[i++] = *lit;
// if (P_pid()==1) printf("rows[%d] = %d\n",i-1,rows[i-1]);
}
freerowsList.clear();
//printf("[%d] rows: %d\n",P_pid(),nrows);
// Rogerio, try to think in something more clear in the future!
/*
* Create a local array consisting of only local free IDs
* */
set<int> setLFNodes;
VIter vit = M_vertexIter(theMesh);
while (pEntity node = VIter_next(vit)){
int ID = get_AppToPETSc_Ordering(EN_id(node));
if ( !getDirichletValue(ID,0) ) {
setLFNodes.insert(ID);
}
}
if (setLFNodes.size()==0){
throw Exception(__LINE__,__FILE__,"No local free nodes.\n");
}
set<int>::iterator setLFN_Iter = setLFNodes.begin();
numLocalIDs = setLFNodes.size();
localIDs = new int[numLocalIDs];
for (i=0; i<numLocalIDs; i++){
localIDs[i] = *setLFN_Iter-1;
setLFN_Iter++;
}
setLFNodes.clear();
return 0;
}
int main(int argc,char **args)
{
Mat *A,B; /* matrix */
PetscErrorCode ierr;
Vec x,y,v,v2,z;
PetscReal rnorm;
PetscInt n = 20; /* size of the matrix */
PetscInt nmat = 3; /* number of matrices */
PetscInt i;
PetscRandom rctx;
MatCompositeType type;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nmat",&nmat,NULL);CHKERRQ(ierr);
/*
Create random matrices
*/
ierr = PetscMalloc1(nmat+3,&A);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,n,n/2,3,NULL,3,NULL,&A[0]);CHKERRQ(ierr);
for (i = 1; i < nmat+1; i++) {
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,n,n,3,NULL,3,NULL,&A[i]);CHKERRQ(ierr);
}
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE,n/2,n,3,NULL,3,NULL,&A[nmat+1]);CHKERRQ(ierr);
for (i = 0; i < nmat+2; i++) {
ierr = MatSetRandom(A[i],rctx);CHKERRQ(ierr);
}
ierr = MatCreateVecs(A[1],&x,&y);CHKERRQ(ierr);
ierr = VecDuplicate(y,&z);CHKERRQ(ierr);
ierr = MatCreateVecs(A[0],&v,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(v,&v2);CHKERRQ(ierr);
ierr = VecSet(x,1.0);CHKERRQ(ierr);
ierr = VecSet(y,0.0);CHKERRQ(ierr);
ierr = MatMult(A[1],x,z);CHKERRQ(ierr);
for (i = 2; i < nmat+1; i++) {
ierr = MatMultAdd(A[i],x,z,z);CHKERRQ(ierr);
}
ierr = MatCreateComposite(PETSC_COMM_WORLD,nmat,A+1,&B);CHKERRQ(ierr);
ierr = MatMultAdd(B,x,y,y);CHKERRQ(ierr);
ierr = VecAXPY(y,-1.0,z);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&rnorm);CHKERRQ(ierr);
if (rnorm > 10000.0*PETSC_MACHINE_EPSILON) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with composite add %g\n",(double)rnorm);CHKERRQ(ierr);
}
ierr = MatCompositeSetMatStructure(B,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); /* default */
ierr = MatCompositeMerge(B);CHKERRQ(ierr);
ierr = MatMult(B,x,y);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecAXPY(y,-1.0,z);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&rnorm);CHKERRQ(ierr);
if (rnorm > 10000.0*PETSC_MACHINE_EPSILON) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with composite add after merge %g\n",(double)rnorm);CHKERRQ(ierr);
}
/*
Test n x n/2 multiplicative composite
*/
ierr = VecSet(v,1.0);CHKERRQ(ierr);
ierr = MatMult(A[0],v,z);CHKERRQ(ierr);
for (i = 1; i < nmat; i++) {
ierr = MatMult(A[i],z,y);CHKERRQ(ierr);
ierr = VecCopy(y,z);CHKERRQ(ierr);
}
ierr = MatCreateComposite(PETSC_COMM_WORLD,nmat,A,&B);CHKERRQ(ierr);
ierr = MatCompositeSetType(B,MAT_COMPOSITE_MULTIPLICATIVE);CHKERRQ(ierr);
ierr = MatCompositeSetMergeType(B,MAT_COMPOSITE_MERGE_LEFT);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* do MatCompositeMerge() if -mat_composite_merge 1 */
ierr = MatMult(B,v,y);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecAXPY(y,-1.0,z);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&rnorm);CHKERRQ(ierr);
if (rnorm > 10000.0*PETSC_MACHINE_EPSILON) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with composite multiplicative %g\n",(double)rnorm);CHKERRQ(ierr);
}
/*
Test n/2 x n multiplicative composite
*/
ierr = VecSet(x,1.0);CHKERRQ(ierr);
ierr = MatMult(A[2],x,z);CHKERRQ(ierr);
for (i = 3; i < nmat+1; i++) {
ierr = MatMult(A[i],z,y);CHKERRQ(ierr);
ierr = VecCopy(y,z);CHKERRQ(ierr);
}
ierr = MatMult(A[nmat+1],z,v);CHKERRQ(ierr);
ierr = MatCreateComposite(PETSC_COMM_WORLD,nmat,A+2,&B);CHKERRQ(ierr);
ierr = MatCompositeSetType(B,MAT_COMPOSITE_MULTIPLICATIVE);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* do MatCompositeMerge() if -mat_composite_merge 1 */
ierr = MatMult(B,x,v2);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecAXPY(v2,-1.0,v);CHKERRQ(ierr);
ierr = VecNorm(v2,NORM_2,&rnorm);CHKERRQ(ierr);
if (rnorm > 10000.0*PETSC_MACHINE_EPSILON) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with composite multiplicative %g\n",(double)rnorm);CHKERRQ(ierr);
}
/*
Test get functions
*/
ierr = MatCreateComposite(PETSC_COMM_WORLD,nmat,A,&B);CHKERRQ(ierr);
ierr = MatCompositeGetNumberMat(B,&n);CHKERRQ(ierr);
if (nmat != n) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with GetNumberMat %D != %D\n",nmat,n);CHKERRQ(ierr);
}
ierr = MatCompositeGetMat(B,0,&A[nmat+2]);CHKERRQ(ierr);
if (A[0] != A[nmat+2]) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with GetMat\n");CHKERRQ(ierr);
}
ierr = MatCompositeGetType(B,&type);CHKERRQ(ierr);
if (type != MAT_COMPOSITE_ADDITIVE) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error with GetType\n");CHKERRQ(ierr);
}
ierr = MatDestroy(&B);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = VecDestroy(&v2);CHKERRQ(ierr);
ierr = VecDestroy(&z);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
for (i = 0; i < nmat+2; i++) {
ierr = MatDestroy(&A[i]);CHKERRQ(ierr);
}
ierr = PetscFree(A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt its,n,Nx=PETSC_DECIDE,Ny=PETSC_DECIDE,nlocal,i;
PetscMPIInt size;
PC pc;
PetscInt mx,my;
Mat A;
GridCtx fine_ctx;
KSP ksp;
PetscBool flg;
PetscInitialize(&argc,&argv,NULL,help);
/* set up discretization matrix for fine grid */
/* ML requires input of fine-grid matrix. It determines nlevels. */
fine_ctx.mx = 9; fine_ctx.my = 9;
ierr = PetscOptionsGetInt(NULL,"-mx",&mx,&flg);CHKERRQ(ierr);
if (flg) fine_ctx.mx = mx;
ierr = PetscOptionsGetInt(NULL,"-my",&my,&flg);CHKERRQ(ierr);
if (flg) fine_ctx.my = my;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Fine grid size %D by %D\n",fine_ctx.mx,fine_ctx.my);CHKERRQ(ierr);
n = fine_ctx.mx*fine_ctx.my;
MPI_Comm_size(PETSC_COMM_WORLD,&size);
ierr = PetscOptionsGetInt(NULL,"-Nx",&Nx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-Ny",&Ny,NULL);CHKERRQ(ierr);
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,fine_ctx.mx,
fine_ctx.my,Nx,Ny,1,1,NULL,NULL,&fine_ctx.da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(fine_ctx.da,&fine_ctx.x);CHKERRQ(ierr);
ierr = VecDuplicate(fine_ctx.x,&fine_ctx.b);CHKERRQ(ierr);
ierr = VecGetLocalSize(fine_ctx.x,&nlocal);CHKERRQ(ierr);
ierr = DMCreateLocalVector(fine_ctx.da,&fine_ctx.localX);CHKERRQ(ierr);
ierr = VecDuplicate(fine_ctx.localX,&fine_ctx.localF);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,nlocal,nlocal,n,n,5,NULL,3,NULL,&A);CHKERRQ(ierr);
ierr = FormJacobian_Grid(&fine_ctx,&A);CHKERRQ(ierr);
/* create linear solver */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCML);CHKERRQ(ierr);
/* set options, then solve system */
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* calls PCSetFromOptions_MG/ML */
for (i=0; i<3; i++) {
if (i<2) { /* test DIFFERENT_NONZERO_PATTERN */
/* set values for rhs vector */
ierr = VecSet(fine_ctx.b,i+1.0);CHKERRQ(ierr);
/* modify A */
ierr = MatShift(A,1.0);CHKERRQ(ierr);
ierr = MatScale(A,2.0);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
} else { /* test SAME_NONZERO_PATTERN */
ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = KSPSolve(ksp,fine_ctx.b,fine_ctx.x);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %D\n",its);CHKERRQ(ierr);
}
/* free data structures */
ierr = VecDestroy(&fine_ctx.x);CHKERRQ(ierr);
ierr = VecDestroy(&fine_ctx.b);CHKERRQ(ierr);
ierr = DMDestroy(&fine_ctx.da);CHKERRQ(ierr);
ierr = VecDestroy(&fine_ctx.localX);CHKERRQ(ierr);
ierr = VecDestroy(&fine_ctx.localF);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode heavyEdgeMatchAgg(IS perm,Mat a_Gmat,PetscInt verbose,PetscCoarsenData **a_locals_llist)
{
PetscErrorCode ierr;
PetscBool isMPI;
MPI_Comm wcomm = ((PetscObject)a_Gmat)->comm;
PetscInt sub_it,kk,n,ix,*idx,*ii,iter,Iend,my0;
PetscMPIInt rank,size;
const PetscInt nloc = a_Gmat->rmap->n,n_iter=6; /* need to figure out how to stop this */
PetscInt *lid_cprowID,*lid_gid;
PetscBool *lid_matched;
Mat_SeqAIJ *matA, *matB=0;
Mat_MPIAIJ *mpimat=0;
PetscScalar one=1.;
PetscCoarsenData *agg_llists = PETSC_NULL,*deleted_list = PETSC_NULL;
Mat cMat,tMat,P;
MatScalar *ap;
PetscMPIInt tag1,tag2;
PetscFunctionBegin;
ierr = MPI_Comm_rank(wcomm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(wcomm, &size);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(a_Gmat, &my0, &Iend);CHKERRQ(ierr);
ierr = PetscCommGetNewTag(wcomm, &tag1);CHKERRQ(ierr);
ierr = PetscCommGetNewTag(wcomm, &tag2);CHKERRQ(ierr);
ierr = PetscMalloc(nloc*sizeof(PetscInt), &lid_gid);CHKERRQ(ierr); /* explicit array needed */
ierr = PetscMalloc(nloc*sizeof(PetscInt), &lid_cprowID);CHKERRQ(ierr);
ierr = PetscMalloc(nloc*sizeof(PetscBool), &lid_matched);CHKERRQ(ierr);
ierr = PetscCDCreate(nloc, &agg_llists);CHKERRQ(ierr);
/* ierr = PetscCDSetChuckSize(agg_llists, nloc+1);CHKERRQ(ierr); */
*a_locals_llist = agg_llists;
ierr = PetscCDCreate(size, &deleted_list);CHKERRQ(ierr);
ierr = PetscCDSetChuckSize(deleted_list, 100);CHKERRQ(ierr);
/* setup 'lid_gid' for scatters and add self to all lists */
for (kk=0;kk<nloc;kk++) {
lid_gid[kk] = kk + my0;
ierr = PetscCDAppendID(agg_llists, kk, my0+kk);CHKERRQ(ierr);
}
/* make a copy of the graph, this gets destroyed in iterates */
ierr = MatDuplicate(a_Gmat,MAT_COPY_VALUES,&cMat);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)a_Gmat, MATMPIAIJ, &isMPI);CHKERRQ(ierr);
iter = 0;
while(iter++ < n_iter) {
PetscScalar *cpcol_gid,*cpcol_max_ew,*cpcol_max_pe,*lid_max_ew;
PetscBool *cpcol_matched;
PetscMPIInt *cpcol_pe,proc;
Vec locMaxEdge,locMaxPE,ghostMaxEdge,ghostMaxPE;
PetscInt nEdges,n_nz_row,jj;
Edge *Edges;
PetscInt gid;
const PetscInt *perm_ix, n_sub_its = 120;
/* get submatrices of cMat */
if (isMPI) {
mpimat = (Mat_MPIAIJ*)cMat->data;
matA = (Mat_SeqAIJ*)mpimat->A->data;
matB = (Mat_SeqAIJ*)mpimat->B->data;
/* force compressed storage of B */
matB->compressedrow.check = PETSC_TRUE;
ierr = MatCheckCompressedRow(mpimat->B,&matB->compressedrow,matB->i,cMat->rmap->n,-1.0);CHKERRQ(ierr);
assert(matB->compressedrow.use);
} else {
matA = (Mat_SeqAIJ*)cMat->data;
}
assert(matA && !matA->compressedrow.use);
assert(matB==0 || matB->compressedrow.use);
/* set max edge on nodes */
ierr = MatGetVecs(cMat, &locMaxEdge, 0);CHKERRQ(ierr);
ierr = MatGetVecs(cMat, &locMaxPE, 0);CHKERRQ(ierr);
/* get 'cpcol_pe' & 'cpcol_gid' & init. 'cpcol_matched' using 'mpimat->lvec' */
if (mpimat) {
Vec vec;
PetscScalar vval;
ierr = MatGetVecs(cMat, &vec, 0);CHKERRQ(ierr);
/* cpcol_pe */
vval = (PetscScalar)(rank);
for (kk=0,gid=my0;kk<nloc;kk++,gid++) {
ierr = VecSetValues(vec, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr); /* set with GID */
}
ierr = VecAssemblyBegin(vec);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec);CHKERRQ(ierr);
ierr = VecScatterBegin(mpimat->Mvctx,vec,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,vec,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(mpimat->lvec, &cpcol_gid);CHKERRQ(ierr); /* get proc ID in 'cpcol_gid' */
ierr = VecGetLocalSize(mpimat->lvec, &n);CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(PetscInt), &cpcol_pe);CHKERRQ(ierr);
for (kk=0;kk<n;kk++) cpcol_pe[kk] = (PetscMPIInt)PetscRealPart(cpcol_gid[kk]);
ierr = VecRestoreArray(mpimat->lvec, &cpcol_gid);CHKERRQ(ierr);
/* cpcol_gid */
for (kk=0,gid=my0;kk<nloc;kk++,gid++) {
vval = (PetscScalar)(gid);
ierr = VecSetValues(vec, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr); /* set with GID */
}
ierr = VecAssemblyBegin(vec);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec);CHKERRQ(ierr);
ierr = VecScatterBegin(mpimat->Mvctx,vec,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,vec,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecDestroy(&vec);CHKERRQ(ierr);
ierr = VecGetArray(mpimat->lvec, &cpcol_gid);CHKERRQ(ierr); /* get proc ID in 'cpcol_gid' */
/* cpcol_matched */
ierr = VecGetLocalSize(mpimat->lvec, &n);CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(PetscBool), &cpcol_matched);CHKERRQ(ierr);
for (kk=0;kk<n;kk++) cpcol_matched[kk] = PETSC_FALSE;
}
/* need an inverse map - locals */
for (kk=0;kk<nloc;kk++) lid_cprowID[kk] = -1;
/* set index into compressed row 'lid_cprowID' */
if (matB) {
ii = matB->compressedrow.i;
for (ix=0; ix<matB->compressedrow.nrows; ix++) {
lid_cprowID[matB->compressedrow.rindex[ix]] = ix;
}
}
/* get removed IS, use '' */
/* if (iter==1) { */
/* PetscInt *lid_rem,idx; */
/* ierr = PetscMalloc(nloc*sizeof(PetscInt), &lid_rem);CHKERRQ(ierr); */
/* for (kk=idx=0;kk<nloc;kk++){ */
/* PetscInt nn,lid=kk; */
/* ii = matA->i; nn = ii[lid+1] - ii[lid]; */
/* if ((ix=lid_cprowID[lid]) != -1) { /\* if I have any ghost neighbors *\/ */
/* ii = matB->compressedrow.i; */
/* nn += ii[ix+1] - ii[ix]; */
/* } */
/* if (nn < 2) { */
/* lid_rem[idx++] = kk + my0; */
/* } */
/* } */
/* ierr = PetscCDSetRemovedIS(agg_llists, wcomm, idx, lid_rem);CHKERRQ(ierr); */
/* ierr = PetscFree(lid_rem);CHKERRQ(ierr); */
/* } */
/* compute 'locMaxEdge' & 'locMaxPE', and create list of edges, count edges' */
for (nEdges=0,kk=0,gid=my0;kk<nloc;kk++,gid++){
PetscReal max_e = 0., tt;
PetscScalar vval;
PetscInt lid = kk;
PetscMPIInt max_pe=rank,pe;
ii = matA->i; n = ii[lid+1] - ii[lid]; idx = matA->j + ii[lid];
ap = matA->a + ii[lid];
for (jj=0; jj<n; jj++) {
PetscInt lidj = idx[jj];
if (lidj != lid && PetscRealPart(ap[jj]) > max_e) max_e = PetscRealPart(ap[jj]);
if (lidj > lid) nEdges++;
}
if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
ap = matB->a + ii[ix];
idx = matB->j + ii[ix];
for (jj=0 ; jj<n ; jj++) {
if ((tt=PetscRealPart(ap[jj])) > max_e) max_e = tt;
nEdges++;
if ((pe=cpcol_pe[idx[jj]]) > max_pe) max_pe = pe;
}
}
vval = max_e;
ierr = VecSetValues(locMaxEdge, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr);
vval = (PetscScalar)max_pe;
ierr = VecSetValues(locMaxPE, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(locMaxEdge);CHKERRQ(ierr);
ierr = VecAssemblyEnd(locMaxEdge);CHKERRQ(ierr);
ierr = VecAssemblyBegin(locMaxPE);CHKERRQ(ierr);
ierr = VecAssemblyEnd(locMaxPE);CHKERRQ(ierr);
/* get 'cpcol_max_ew' & 'cpcol_max_pe' */
if (mpimat) {
ierr = VecDuplicate(mpimat->lvec, &ghostMaxEdge);CHKERRQ(ierr);
ierr = VecScatterBegin(mpimat->Mvctx,locMaxEdge,ghostMaxEdge,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,locMaxEdge,ghostMaxEdge,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(ghostMaxEdge, &cpcol_max_ew);CHKERRQ(ierr);
ierr = VecDuplicate(mpimat->lvec, &ghostMaxPE);CHKERRQ(ierr);
ierr = VecScatterBegin(mpimat->Mvctx,locMaxPE,ghostMaxPE,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,locMaxPE,ghostMaxPE,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(ghostMaxPE, &cpcol_max_pe);CHKERRQ(ierr);
}
/* setup sorted list of edges */
ierr = PetscMalloc(nEdges*sizeof(Edge), &Edges);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &perm_ix);CHKERRQ(ierr);
for (nEdges=n_nz_row=kk=0;kk<nloc;kk++){
PetscInt nn, lid = perm_ix[kk];
ii = matA->i; nn = n = ii[lid+1] - ii[lid]; idx = matA->j + ii[lid];
ap = matA->a + ii[lid];
for (jj=0; jj<n; jj++) {
PetscInt lidj = idx[jj]; assert(PetscRealPart(ap[jj])>0.);
if (lidj > lid) {
Edges[nEdges].lid0 = lid;
Edges[nEdges].gid1 = lidj + my0;
Edges[nEdges].cpid1 = -1;
Edges[nEdges].weight = PetscRealPart(ap[jj]);
nEdges++;
}
}
if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
ap = matB->a + ii[ix];
idx = matB->j + ii[ix];
nn += n;
for (jj=0 ; jj<n ; jj++) {
assert(PetscRealPart(ap[jj])>0.);
Edges[nEdges].lid0 = lid;
Edges[nEdges].gid1 = (PetscInt)PetscRealPart(cpcol_gid[idx[jj]]);
Edges[nEdges].cpid1 = idx[jj];
Edges[nEdges].weight = PetscRealPart(ap[jj]);
nEdges++;
}
}
if (nn > 1) n_nz_row++;
else if (iter == 1){
/* should select this because it is technically in the MIS but lets not */
ierr = PetscCDRemoveAll(agg_llists, lid);CHKERRQ(ierr);
}
}
ierr = ISRestoreIndices(perm,&perm_ix);CHKERRQ(ierr);
qsort(Edges, nEdges, sizeof(Edge), gamg_hem_compare);
/* projection matrix */
ierr = MatCreateAIJ(wcomm, nloc, nloc, PETSC_DETERMINE, PETSC_DETERMINE, 1, 0, 1, 0, &P);CHKERRQ(ierr);
/* clear matched flags */
for (kk=0;kk<nloc;kk++) lid_matched[kk] = PETSC_FALSE;
/* process - communicate - process */
for (sub_it=0;sub_it<n_sub_its;sub_it++){
PetscInt nactive_edges;
ierr = VecGetArray(locMaxEdge, &lid_max_ew);CHKERRQ(ierr);
for (kk=nactive_edges=0;kk<nEdges;kk++){
/* HEM */
const Edge *e = &Edges[kk];
const PetscInt lid0=e->lid0,gid1=e->gid1,cpid1=e->cpid1,gid0=lid0+my0,lid1=gid1-my0;
PetscBool isOK = PETSC_TRUE;
/* skip if either (local) vertex is done already */
if (lid_matched[lid0] || (gid1>=my0 && gid1<Iend && lid_matched[gid1-my0])) {
continue;
}
/* skip if ghost vertex is done */
if (cpid1 != -1 && cpcol_matched[cpid1]) {
continue;
}
nactive_edges++;
/* skip if I have a bigger edge someplace (lid_max_ew gets updated) */
if (PetscRealPart(lid_max_ew[lid0]) > e->weight + 1.e-12) {
continue;
}
if (cpid1 == -1) {
if (PetscRealPart(lid_max_ew[lid1]) > e->weight + 1.e-12) {
continue;
}
} else {
/* see if edge might get matched on other proc */
PetscReal g_max_e = PetscRealPart(cpcol_max_ew[cpid1]);
if (g_max_e > e->weight + 1.e-12) {
continue;
} else if (e->weight > g_max_e - 1.e-12 && (PetscMPIInt)PetscRealPart(cpcol_max_pe[cpid1]) > rank) {
/* check for max_e == to this edge and larger processor that will deal with this */
continue;
}
}
/* check ghost for v0 */
if (isOK){
PetscReal max_e,ew;
if ((ix=lid_cprowID[lid0]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
ap = matB->a + ii[ix];
idx = matB->j + ii[ix];
for (jj=0 ; jj<n && isOK; jj++) {
PetscInt lidj = idx[jj];
if (cpcol_matched[lidj]) continue;
ew = PetscRealPart(ap[jj]); max_e = PetscRealPart(cpcol_max_ew[lidj]);
/* check for max_e == to this edge and larger processor that will deal with this */
if (ew > max_e - 1.e-12 && ew > PetscRealPart(lid_max_ew[lid0]) - 1.e-12 && (PetscMPIInt)PetscRealPart(cpcol_max_pe[lidj]) > rank){
isOK = PETSC_FALSE;
}
}
}
/* for v1 */
if (cpid1 == -1 && isOK){
if ((ix=lid_cprowID[lid1]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
ap = matB->a + ii[ix];
idx = matB->j + ii[ix];
for (jj=0 ; jj<n && isOK ; jj++) {
PetscInt lidj = idx[jj];
if (cpcol_matched[lidj]) continue;
ew = PetscRealPart(ap[jj]); max_e = PetscRealPart(cpcol_max_ew[lidj]);
/* check for max_e == to this edge and larger processor that will deal with this */
if (ew > max_e - 1.e-12 && ew > PetscRealPart(lid_max_ew[lid1]) - 1.e-12 && (PetscMPIInt)PetscRealPart(cpcol_max_pe[lidj]) > rank) {
isOK = PETSC_FALSE;
}
}
}
}
}
/* do it */
if (isOK){
if (cpid1 == -1) {
lid_matched[lid1] = PETSC_TRUE; /* keep track of what we've done this round */
ierr = PetscCDAppendRemove(agg_llists, lid0, lid1);CHKERRQ(ierr);
} else if (sub_it != n_sub_its-1) {
/* add gid1 to list of ghost deleted by me -- I need their children */
proc = cpcol_pe[cpid1];
cpcol_matched[cpid1] = PETSC_TRUE; /* messing with VecGetArray array -- needed??? */
ierr = PetscCDAppendID(deleted_list, proc, cpid1);CHKERRQ(ierr); /* cache to send messages */
ierr = PetscCDAppendID(deleted_list, proc, lid0);CHKERRQ(ierr);
} else {
continue;
}
lid_matched[lid0] = PETSC_TRUE; /* keep track of what we've done this round */
/* set projection */
ierr = MatSetValues(P,1,&gid0,1,&gid0,&one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(P,1,&gid1,1,&gid0,&one,INSERT_VALUES);CHKERRQ(ierr);
} /* matched */
} /* edge loop */
/* deal with deleted ghost on first pass */
if (size>1 && sub_it != n_sub_its-1){
PetscCDPos pos; PetscBool ise = PETSC_FALSE;
PetscInt nSend1, **sbuffs1,nSend2;
#define REQ_BF_SIZE 100
MPI_Request *sreqs2[REQ_BF_SIZE],*rreqs2[REQ_BF_SIZE];
MPI_Status status;
/* send request */
for (proc=0,nSend1=0;proc<size;proc++){
ierr = PetscCDEmptyAt(deleted_list,proc,&ise);CHKERRQ(ierr);
if (!ise) nSend1++;
}
ierr = PetscMalloc(nSend1*sizeof(PetscInt*), &sbuffs1);CHKERRQ(ierr);
/* ierr = PetscMalloc4(nSend1, PetscInt*, sbuffs1, nSend1, PetscInt*, rbuffs1, nSend1, MPI_Request*, sreqs1, nSend1, MPI_Request*, rreqs1);CHKERRQ(ierr); */
/* PetscFree4(sbuffs1,rbuffs1,sreqs1,rreqs1); */
for (proc=0,nSend1=0;proc<size;proc++){
/* count ghosts */
ierr = PetscCDSizeAt(deleted_list,proc,&n);CHKERRQ(ierr);
if (n>0){
#define CHUNCK_SIZE 100
PetscInt *sbuff,*pt;
MPI_Request *request;
assert(n%2==0);
n /= 2;
ierr = PetscMalloc((2 + 2*n + n*CHUNCK_SIZE)*sizeof(PetscInt) + 2*sizeof(MPI_Request), &sbuff);CHKERRQ(ierr);
/* PetscMalloc4(2+2*n,PetscInt,sbuffs1[nSend1],n*CHUNCK_SIZE,PetscInt,rbuffs1[nSend1],1,MPI_Request,rreqs2[nSend1],1,MPI_Request,sreqs2[nSend1]); */
/* save requests */
sbuffs1[nSend1] = sbuff;
request = (MPI_Request*)sbuff;
sbuff = pt = (PetscInt*)(request+1);
*pt++ = n; *pt++ = rank;
ierr = PetscCDGetHeadPos(deleted_list,proc,&pos);CHKERRQ(ierr);
while(pos){
PetscInt lid0, cpid, gid;
ierr = PetscLLNGetID(pos, &cpid);CHKERRQ(ierr);
gid = (PetscInt)PetscRealPart(cpcol_gid[cpid]);
ierr = PetscCDGetNextPos(deleted_list,proc,&pos);CHKERRQ(ierr);
ierr = PetscLLNGetID(pos, &lid0);CHKERRQ(ierr);
ierr = PetscCDGetNextPos(deleted_list,proc,&pos);CHKERRQ(ierr);
*pt++ = gid; *pt++ = lid0;
}
/* send request tag1 [n, proc, n*[gid1,lid0] ] */
ierr = MPI_Isend(sbuff, 2*n+2, MPIU_INT, proc, tag1, wcomm, request);CHKERRQ(ierr);
/* post recieve */
request = (MPI_Request*)pt;
rreqs2[nSend1] = request; /* cache recv request */
pt = (PetscInt*)(request+1);
ierr = MPI_Irecv(pt, n*CHUNCK_SIZE, MPIU_INT, proc, tag2, wcomm, request);CHKERRQ(ierr);
/* clear list */
ierr = PetscCDRemoveAll(deleted_list, proc);CHKERRQ(ierr);
nSend1++;
}
}
/* recieve requests, send response, clear lists */
kk = nactive_edges;
ierr = MPI_Allreduce(&kk,&nactive_edges,1,MPIU_INT,MPI_SUM,wcomm);CHKERRQ(ierr); /* not correct syncronization and global */
nSend2 = 0;
while(1){
#define BF_SZ 10000
PetscMPIInt flag,count;
PetscInt rbuff[BF_SZ],*pt,*pt2,*pt3,count2,*sbuff,count3;
MPI_Request *request;
ierr = MPI_Iprobe(MPI_ANY_SOURCE, tag1, wcomm, &flag, &status);CHKERRQ(ierr);
if (!flag) break;
ierr = MPI_Get_count(&status, MPIU_INT, &count);CHKERRQ(ierr);
if (count > BF_SZ) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"buffer too small for recieve: %d",count);
proc = status.MPI_SOURCE;
/* recieve request tag1 [n, proc, n*[gid1,lid0] ] */
ierr = MPI_Recv(rbuff, count, MPIU_INT, proc, tag1, wcomm, &status);CHKERRQ(ierr);
/* count sends */
pt = rbuff; count3 = count2 = 0;
n = *pt++; kk = *pt++; assert(kk==proc);
while(n--){
PetscInt gid1=*pt++, lid1=gid1-my0; kk=*pt++; assert(lid1>=0 && lid1<nloc);
if (lid_matched[lid1]){
PetscPrintf(PETSC_COMM_SELF,"\t *** [%d]%s %d) ERROR recieved deleted gid %d, deleted by (lid) %d from proc %d\n",rank,__FUNCT__,sub_it,gid1,kk);
PetscSleep(1);
}
assert(!lid_matched[lid1]);
lid_matched[lid1] = PETSC_TRUE; /* keep track of what we've done this round */
ierr = PetscCDSizeAt(agg_llists, lid1, &kk);CHKERRQ(ierr);
count2 += kk + 2;
count3++; /* number of verts requested (n) */
}
assert(pt-rbuff==count);
if (count2 > count3*CHUNCK_SIZE) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Irecv will be too small: %d",count2);
/* send tag2 *[lid0, n, n*[gid] ] */
ierr = PetscMalloc(count2*sizeof(PetscInt) + sizeof(MPI_Request), &sbuff);CHKERRQ(ierr);
request = (MPI_Request*)sbuff;
sreqs2[nSend2++] = request; /* cache request */
if (nSend2==REQ_BF_SIZE) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"buffer too small for requests: %d",nSend2);
pt2 = sbuff = (PetscInt*)(request+1);
pt = rbuff;
n = *pt++; kk = *pt++; assert(kk==proc);
while(n--){
/* read [n, proc, n*[gid1,lid0] */
PetscInt gid1=*pt++, lid1=gid1-my0, lid0=*pt++; assert(lid1>=0 && lid1<nloc);
/* write [lid0, n, n*[gid] ] */
*pt2++ = lid0;
pt3 = pt2++; /* save pointer for later */
/* for (pos=PetscCDGetHeadPos(agg_llists,lid1) ; pos ; pos=PetscCDGetNextPos(agg_llists,lid1,pos)){ */
ierr = PetscCDGetHeadPos(agg_llists,lid1,&pos);CHKERRQ(ierr);
while(pos){
PetscInt gid;
ierr = PetscLLNGetID(pos, &gid);CHKERRQ(ierr);
ierr = PetscCDGetNextPos(agg_llists,lid1,&pos);CHKERRQ(ierr);
*pt2++ = gid;
}
*pt3 = (pt2-pt3)-1;
/* clear list */
ierr = PetscCDRemoveAll(agg_llists, lid1);CHKERRQ(ierr);
}
assert(pt2-sbuff==count2); assert(pt-rbuff==count);
/* send requested data tag2 *[lid0, n, n*[gid1] ] */
ierr = MPI_Isend(sbuff, count2, MPIU_INT, proc, tag2, wcomm, request);CHKERRQ(ierr);
}
/* recieve tag2 *[lid0, n, n*[gid] ] */
for (kk=0;kk<nSend1;kk++){
PetscMPIInt count;
MPI_Request *request;
PetscInt *pt, *pt2;
request = rreqs2[kk]; /* no need to free -- buffer is in 'sbuffs1' */
ierr = MPI_Wait(request, &status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status, MPIU_INT, &count);CHKERRQ(ierr);
pt = pt2 = (PetscInt*)(request+1);
while(pt-pt2 < count){
PetscInt lid0 = *pt++, n = *pt++; assert(lid0>=0 && lid0<nloc);
while(n--){
PetscInt gid1 = *pt++;
ierr = PetscCDAppendID(agg_llists, lid0, gid1);CHKERRQ(ierr);
}
}
assert(pt-pt2==count);
}
/* wait for tag1 isends */
while(nSend1--){
MPI_Request *request;
request = (MPI_Request*)sbuffs1[nSend1];
ierr = MPI_Wait(request, &status);CHKERRQ(ierr);
ierr = PetscFree(request);CHKERRQ(ierr);
}
ierr = PetscFree(sbuffs1);CHKERRQ(ierr);
/* wait for tag2 isends */
while(nSend2--){
MPI_Request *request = sreqs2[nSend2];
ierr = MPI_Wait(request, &status);CHKERRQ(ierr);
ierr = PetscFree(request);CHKERRQ(ierr);
}
ierr = VecRestoreArray(ghostMaxEdge, &cpcol_max_ew);CHKERRQ(ierr);
ierr = VecRestoreArray(ghostMaxPE, &cpcol_max_pe);CHKERRQ(ierr);
/* get 'cpcol_matched' - use locMaxPE, ghostMaxEdge, cpcol_max_ew */
for (kk=0,gid=my0;kk<nloc;kk++,gid++) {
PetscScalar vval = lid_matched[kk] ? 1.0 : 0.0;
ierr = VecSetValues(locMaxPE, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr); /* set with GID */
}
ierr = VecAssemblyBegin(locMaxPE);CHKERRQ(ierr);
ierr = VecAssemblyEnd(locMaxPE);CHKERRQ(ierr);
ierr = VecScatterBegin(mpimat->Mvctx,locMaxPE,ghostMaxEdge,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,locMaxPE,ghostMaxEdge,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(ghostMaxEdge, &cpcol_max_ew);CHKERRQ(ierr);
ierr = VecGetLocalSize(mpimat->lvec, &n);CHKERRQ(ierr);
for (kk=0;kk<n;kk++) {
cpcol_matched[kk] = (PetscBool)(PetscRealPart(cpcol_max_ew[kk]) != 0.0);
}
ierr = VecRestoreArray(ghostMaxEdge, &cpcol_max_ew);CHKERRQ(ierr);
} /* size > 1 */
/* compute 'locMaxEdge' */
ierr = VecRestoreArray(locMaxEdge, &lid_max_ew);CHKERRQ(ierr);
for (kk=0,gid=my0;kk<nloc;kk++,gid++){
PetscReal max_e = 0.,tt;
PetscScalar vval;
PetscInt lid = kk;
if (lid_matched[lid]) vval = 0.;
else {
ii = matA->i; n = ii[lid+1] - ii[lid]; idx = matA->j + ii[lid];
ap = matA->a + ii[lid];
for (jj=0; jj<n; jj++) {
PetscInt lidj = idx[jj];
if (lid_matched[lidj]) continue; /* this is new - can change local max */
if (lidj != lid && PetscRealPart(ap[jj]) > max_e) max_e = PetscRealPart(ap[jj]);
}
if (lid_cprowID && (ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
ap = matB->a + ii[ix];
idx = matB->j + ii[ix];
for (jj=0 ; jj<n ; jj++) {
PetscInt lidj = idx[jj];
if (cpcol_matched[lidj]) continue;
if ((tt=PetscRealPart(ap[jj])) > max_e) max_e = tt;
}
}
}
vval = (PetscScalar)max_e;
ierr = VecSetValues(locMaxEdge, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr); /* set with GID */
}
ierr = VecAssemblyBegin(locMaxEdge);CHKERRQ(ierr);
ierr = VecAssemblyEnd(locMaxEdge);CHKERRQ(ierr);
if (size>1 && sub_it != n_sub_its-1){
/* compute 'cpcol_max_ew' */
ierr = VecScatterBegin(mpimat->Mvctx,locMaxEdge,ghostMaxEdge,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,locMaxEdge,ghostMaxEdge,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(ghostMaxEdge, &cpcol_max_ew);CHKERRQ(ierr);
ierr = VecGetArray(locMaxEdge, &lid_max_ew);CHKERRQ(ierr);
/* compute 'cpcol_max_pe' */
for (kk=0,gid=my0;kk<nloc;kk++,gid++){
PetscInt lid = kk;
PetscReal ew,v1_max_e,v0_max_e=PetscRealPart(lid_max_ew[lid]);
PetscScalar vval;
PetscMPIInt max_pe=rank,pe;
if (lid_matched[lid]) vval = (PetscScalar)rank;
else if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
ap = matB->a + ii[ix];
idx = matB->j + ii[ix];
for (jj=0 ; jj<n ; jj++) {
PetscInt lidj = idx[jj];
if (cpcol_matched[lidj]) continue;
ew = PetscRealPart(ap[jj]); v1_max_e = PetscRealPart(cpcol_max_ew[lidj]);
/* get max pe that has a max_e == to this edge w */
if ((pe=cpcol_pe[idx[jj]]) > max_pe && ew > v1_max_e - 1.e-12 && ew > v0_max_e - 1.e-12) max_pe = pe;
assert(ew < v0_max_e + 1.e-12 && ew < v1_max_e + 1.e-12);
}
vval = (PetscScalar)max_pe;
}
ierr = VecSetValues(locMaxPE, 1, &gid, &vval, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(locMaxPE);CHKERRQ(ierr);
ierr = VecAssemblyEnd(locMaxPE);CHKERRQ(ierr);
ierr = VecScatterBegin(mpimat->Mvctx,locMaxPE,ghostMaxPE,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,locMaxPE,ghostMaxPE,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(ghostMaxPE, &cpcol_max_pe);CHKERRQ(ierr);
ierr = VecRestoreArray(locMaxEdge, &lid_max_ew);CHKERRQ(ierr);
} /* deal with deleted ghost */
if (verbose>2) PetscPrintf(wcomm,"\t[%d]%s %d.%d: %d active edges.\n",
rank,__FUNCT__,iter,sub_it,nactive_edges);
if (!nactive_edges) break;
} /* sub_it loop */
/* clean up iteration */
ierr = PetscFree(Edges);CHKERRQ(ierr);
if (mpimat){
ierr = VecRestoreArray(ghostMaxEdge, &cpcol_max_ew);CHKERRQ(ierr);
ierr = VecDestroy(&ghostMaxEdge);CHKERRQ(ierr);
ierr = VecRestoreArray(ghostMaxPE, &cpcol_max_pe);CHKERRQ(ierr);
ierr = VecDestroy(&ghostMaxPE);CHKERRQ(ierr);
ierr = PetscFree(cpcol_pe);CHKERRQ(ierr);
ierr = PetscFree(cpcol_matched);CHKERRQ(ierr);
}
ierr = VecDestroy(&locMaxEdge);CHKERRQ(ierr);
ierr = VecDestroy(&locMaxPE);CHKERRQ(ierr);
if (mpimat){
ierr = VecRestoreArray(mpimat->lvec, &cpcol_gid);CHKERRQ(ierr);
}
/* create next G if needed */
if (iter == n_iter) { /* hard wired test - need to look at full surrounded nodes or something */
ierr = MatDestroy(&P);CHKERRQ(ierr);
ierr = MatDestroy(&cMat);CHKERRQ(ierr);
break;
} else {
Vec diag;
/* add identity for unmatched vertices so they stay alive */
for (kk=0,gid=my0;kk<nloc;kk++,gid++){
if (!lid_matched[kk]) {
gid = kk+my0;
ierr = MatGetRow(cMat,gid,&n,0,0);CHKERRQ(ierr);
if (n>1){
ierr = MatSetValues(P,1,&gid,1,&gid,&one,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatRestoreRow(cMat,gid,&n,0,0);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(P,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(P,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* project to make new graph with colapsed edges */
ierr = MatPtAP(cMat,P,MAT_INITIAL_MATRIX,1.0,&tMat);CHKERRQ(ierr);
ierr = MatDestroy(&P);CHKERRQ(ierr);
ierr = MatDestroy(&cMat);CHKERRQ(ierr);
cMat = tMat;
ierr = MatGetVecs(cMat, &diag, 0);CHKERRQ(ierr);
ierr = MatGetDiagonal(cMat, diag);CHKERRQ(ierr); /* effectively PCJACOBI */
ierr = VecReciprocal(diag);CHKERRQ(ierr);
ierr = VecSqrtAbs(diag);CHKERRQ(ierr);
ierr = MatDiagonalScale(cMat, diag, diag);CHKERRQ(ierr);
ierr = VecDestroy(&diag);CHKERRQ(ierr);
}
} /* coarsen iterator */
/* make fake matrix */
if (size>1){
Mat mat;
PetscCDPos pos;
PetscInt gid, NN, MM, jj = 0, mxsz = 0;
for (kk=0;kk<nloc;kk++){
ierr = PetscCDSizeAt(agg_llists, kk, &jj);CHKERRQ(ierr);
if (jj > mxsz) mxsz = jj;
}
ierr = MatGetSize(a_Gmat, &MM, &NN);CHKERRQ(ierr);
if (mxsz > MM-nloc) mxsz = MM-nloc;
ierr = MatCreateAIJ(wcomm, nloc, nloc,PETSC_DETERMINE, PETSC_DETERMINE,0, 0, mxsz, 0, &mat);CHKERRQ(ierr);
/* */
for (kk=0,gid=my0;kk<nloc;kk++,gid++){
/* for (pos=PetscCDGetHeadPos(agg_llists,kk) ; pos ; pos=PetscCDGetNextPos(agg_llists,kk,pos)){ */
ierr = PetscCDGetHeadPos(agg_llists,kk,&pos);CHKERRQ(ierr);
while(pos){
PetscInt gid1;
ierr = PetscLLNGetID(pos, &gid1);CHKERRQ(ierr);
ierr = PetscCDGetNextPos(agg_llists,kk,&pos);CHKERRQ(ierr);
if (gid1 < my0 || gid1 >= my0+nloc) {
ierr = MatSetValues(mat,1,&gid,1,&gid1,&one,ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscCDSetMat(agg_llists, mat);CHKERRQ(ierr);
}
ierr = PetscFree(lid_cprowID);CHKERRQ(ierr);
ierr = PetscFree(lid_gid);CHKERRQ(ierr);
ierr = PetscFree(lid_matched);CHKERRQ(ierr);
ierr = PetscCDDestroy(deleted_list);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat C,A;
PetscInt i,j,m = 3,n = 2,rstart,rend;
PetscMPIInt size,rank;
PetscErrorCode ierr;
PetscScalar v;
IS isrow,iscol;
PetscBool flg;
char type[256];
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
n = 2*size;
ierr = PetscStrcpy(type,MATSAME);CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-mat_type",type,256,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscStrcmp(type,MATMPIDENSE,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatCreateDense(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE, m*n,m*n,PETSC_NULL,&C);CHKERRQ(ierr);
} else {
ierr = MatCreateAIJ(PETSC_COMM_WORLD,PETSC_DECIDE,PETSC_DECIDE, m*n,m*n,PETSC_DECIDE,PETSC_NULL,PETSC_DECIDE,PETSC_NULL,&C);CHKERRQ(ierr);
}
/*
This is JUST to generate a nice test matrix, all processors fill up
the entire matrix. This is not something one would ever do in practice.
*/
for (i=0; i<m*n; i++) {
for (j=0; j<m*n; j++) {
v = i + j + 1;
ierr = MatSetValues(C,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/*
Generate a new matrix consisting of every second row and column of
the original matrix
*/
ierr = MatGetOwnershipRange(C,&rstart,&rend);CHKERRQ(ierr);
/* Create parallel IS with the rows we want on THIS processor */
ierr = ISCreateStride(PETSC_COMM_WORLD,(rend-rstart)/2,rstart,2,&isrow);CHKERRQ(ierr);
/* Create parallel IS with the rows we want on THIS processor (same as rows for now) */
ierr = ISCreateStride(PETSC_COMM_WORLD,(rend-rstart)/2,rstart,2,&iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrix(C,isrow,iscol,MAT_INITIAL_MATRIX,&A);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatGetSubMatrix(C,isrow,iscol,MAT_REUSE_MATRIX,&A);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*
DMDAGetFaceInterpolation - Gets the interpolation for a face based coarse space
*/
PetscErrorCode DMDAGetFaceInterpolation(DM da,PC_Exotic *exotic,Mat Aglobal,MatReuse reuse,Mat *P)
{
PetscErrorCode ierr;
PetscInt dim,i,j,k,m,n,p,dof,Nint,Nface,Nwire,Nsurf,*Iint,*Isurf,cint = 0,csurf = 0,istart,jstart,kstart,*II,N,c = 0;
PetscInt mwidth,nwidth,pwidth,cnt,mp,np,pp,Ntotal,gl[6],*globals,Ng,*IIint,*IIsurf,Nt;
Mat Xint, Xsurf,Xint_tmp;
IS isint,issurf,is,row,col;
ISLocalToGlobalMapping ltg;
MPI_Comm comm;
Mat A,Aii,Ais,Asi,*Aholder,iAii;
MatFactorInfo info;
PetscScalar *xsurf,*xint;
#if defined(PETSC_USE_DEBUG_foo)
PetscScalar tmp;
#endif
PetscTable ht;
PetscFunctionBegin;
ierr = DMDAGetInfo(da,&dim,0,0,0,&mp,&np,&pp,&dof,0,0,0,0,0);
CHKERRQ(ierr);
if (dof != 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only for single field problems");
if (dim != 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only coded for 3d problems");
ierr = DMDAGetCorners(da,0,0,0,&m,&n,&p);
CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&istart,&jstart,&kstart,&mwidth,&nwidth,&pwidth);
CHKERRQ(ierr);
istart = istart ? -1 : 0;
jstart = jstart ? -1 : 0;
kstart = kstart ? -1 : 0;
/*
the columns of P are the interpolation of each coarse grid point (one for each vertex and edge)
to all the local degrees of freedom (this includes the vertices, edges and faces).
Xint are the subset of the interpolation into the interior
Xface are the interpolation onto faces but not into the interior
Xsurf are the interpolation onto the vertices and edges (the surfbasket)
Xint
Symbolically one could write P = (Xface) after interchanging the rows to match the natural ordering on the domain
Xsurf
*/
N = (m - istart)*(n - jstart)*(p - kstart);
Nint = (m-2-istart)*(n-2-jstart)*(p-2-kstart);
Nface = 2*((m-2-istart)*(n-2-jstart) + (m-2-istart)*(p-2-kstart) + (n-2-jstart)*(p-2-kstart));
Nwire = 4*((m-2-istart) + (n-2-jstart) + (p-2-kstart)) + 8;
Nsurf = Nface + Nwire;
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nint,6,NULL,&Xint);
CHKERRQ(ierr);
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nsurf,6,NULL,&Xsurf);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/*
Require that all 12 edges and 6 faces have at least one grid point. Otherwise some of the columns of
Xsurf will be all zero (thus making the coarse matrix singular).
*/
if (m-istart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in X direction must be at least 3");
if (n-jstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Y direction must be at least 3");
if (p-kstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Z direction must be at least 3");
cnt = 0;
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 0*Nsurf] = 1;
}
for (k=1; k<p-1-kstart; k++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 1*Nsurf] = 1;
for (j=1; j<n-1-jstart; j++) {
xsurf[cnt++ + 2*Nsurf] = 1;
/* these are the interior nodes */
xsurf[cnt++ + 3*Nsurf] = 1;
}
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 4*Nsurf] = 1;
}
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 5*Nsurf] = 1;
}
#if defined(PETSC_USE_DEBUG_foo)
for (i=0; i<Nsurf; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xsurf[i+j*Nsurf];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xsurf interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
#endif
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/* ierr = MatView(Xsurf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
/*
I are the indices for all the needed vertices (in global numbering)
Iint are the indices for the interior values, I surf for the surface values
(This is just for the part of the global matrix obtained with MatGetSubMatrix(), it
is NOT the local DMDA ordering.)
IIint and IIsurf are the same as the Iint, Isurf except they are in the global numbering
*/
#define Endpoint(a,start,b) (a == 0 || a == (b-1-start))
ierr = PetscMalloc3(N,&II,Nint,&Iint,Nsurf,&Isurf);
CHKERRQ(ierr);
ierr = PetscMalloc2(Nint,&IIint,Nsurf,&IIsurf);
CHKERRQ(ierr);
for (k=0; k<p-kstart; k++) {
for (j=0; j<n-jstart; j++) {
for (i=0; i<m-istart; i++) {
II[c++] = i + j*mwidth + k*mwidth*nwidth;
if (!Endpoint(i,istart,m) && !Endpoint(j,jstart,n) && !Endpoint(k,kstart,p)) {
IIint[cint] = i + j*mwidth + k*mwidth*nwidth;
Iint[cint++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
} else {
IIsurf[csurf] = i + j*mwidth + k*mwidth*nwidth;
Isurf[csurf++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
}
}
}
}
if (c != N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"c != N");
if (cint != Nint) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"cint != Nint");
if (csurf != Nsurf) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"csurf != Nsurf");
ierr = DMGetLocalToGlobalMapping(da,<g);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,N,II,II);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nint,IIint,IIint);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nsurf,IIsurf,IIsurf);
CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)da,&comm);
CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,N,II,PETSC_COPY_VALUES,&is);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nint,Iint,PETSC_COPY_VALUES,&isint);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nsurf,Isurf,PETSC_COPY_VALUES,&issurf);
CHKERRQ(ierr);
ierr = PetscFree3(II,Iint,Isurf);
CHKERRQ(ierr);
ierr = ISSort(is);
CHKERRQ(ierr);
ierr = MatGetSubMatrices(Aglobal,1,&is,&is,MAT_INITIAL_MATRIX,&Aholder);
CHKERRQ(ierr);
A = *Aholder;
ierr = PetscFree(Aholder);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,isint,MAT_INITIAL_MATRIX,&Aii);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,issurf,MAT_INITIAL_MATRIX,&Ais);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,issurf,isint,MAT_INITIAL_MATRIX,&Asi);
CHKERRQ(ierr);
/*
Solve for the interpolation onto the interior Xint
*/
ierr = MatMatMult(Ais,Xsurf,MAT_INITIAL_MATRIX,PETSC_DETERMINE,&Xint_tmp);
CHKERRQ(ierr);
ierr = MatScale(Xint_tmp,-1.0);
CHKERRQ(ierr);
if (exotic->directSolve) {
ierr = MatGetFactor(Aii,MATSOLVERPETSC,MAT_FACTOR_LU,&iAii);
CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);
CHKERRQ(ierr);
ierr = MatGetOrdering(Aii,MATORDERINGND,&row,&col);
CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(iAii,Aii,row,col,&info);
CHKERRQ(ierr);
ierr = ISDestroy(&row);
CHKERRQ(ierr);
ierr = ISDestroy(&col);
CHKERRQ(ierr);
ierr = MatLUFactorNumeric(iAii,Aii,&info);
CHKERRQ(ierr);
ierr = MatMatSolve(iAii,Xint_tmp,Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&iAii);
CHKERRQ(ierr);
} else {
Vec b,x;
PetscScalar *xint_tmp;
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&x);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&b);
CHKERRQ(ierr);
ierr = KSPSetOperators(exotic->ksp,Aii,Aii);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = VecPlaceArray(x,xint+i*Nint);
CHKERRQ(ierr);
ierr = VecPlaceArray(b,xint_tmp+i*Nint);
CHKERRQ(ierr);
ierr = KSPSolve(exotic->ksp,b,x);
CHKERRQ(ierr);
ierr = VecResetArray(x);
CHKERRQ(ierr);
ierr = VecResetArray(b);
CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
}
ierr = MatDestroy(&Xint_tmp);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
for (i=0; i<Nint; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xint[i+j*Nint];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xint interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
/* ierr =MatView(Xint,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
#endif
/* total faces */
Ntotal = mp*np*(pp+1) + mp*pp*(np+1) + np*pp*(mp+1);
/*
For each vertex, edge, face on process (in the same orderings as used above) determine its local number including ghost points
*/
cnt = 0;
{
gl[cnt++] = mwidth+1;
}
{
{
gl[cnt++] = mwidth*nwidth+1;
}
{
gl[cnt++] = mwidth*nwidth + mwidth; /* these are the interior nodes */ gl[cnt++] = mwidth*nwidth + mwidth+m-istart-1;
}
{
gl[cnt++] = mwidth*nwidth+mwidth*(n-jstart-1)+1;
}
}
{
gl[cnt++] = mwidth*nwidth*(p-kstart-1) + mwidth+1;
}
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* convert that to global numbering and get them on all processes */
ierr = ISLocalToGlobalMappingApply(ltg,6,gl,gl);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
ierr = PetscMalloc1(6*mp*np*pp,&globals);
CHKERRQ(ierr);
ierr = MPI_Allgather(gl,6,MPIU_INT,globals,6,MPIU_INT,PetscObjectComm((PetscObject)da));
CHKERRQ(ierr);
/* Number the coarse grid points from 0 to Ntotal */
ierr = MatGetSize(Aglobal,&Nt,NULL);
CHKERRQ(ierr);
ierr = PetscTableCreate(Ntotal/3,Nt+1,&ht);
CHKERRQ(ierr);
for (i=0; i<6*mp*np*pp; i++) {
ierr = PetscTableAddCount(ht,globals[i]+1);
CHKERRQ(ierr);
}
ierr = PetscTableGetCount(ht,&cnt);
CHKERRQ(ierr);
if (cnt != Ntotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Hash table size %D not equal to total number coarse grid points %D",cnt,Ntotal);
ierr = PetscFree(globals);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = PetscTableFind(ht,gl[i]+1,&gl[i]);
CHKERRQ(ierr);
gl[i]--;
}
ierr = PetscTableDestroy(&ht);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* construct global interpolation matrix */
ierr = MatGetLocalSize(Aglobal,&Ng,NULL);
CHKERRQ(ierr);
if (reuse == MAT_INITIAL_MATRIX) {
ierr = MatCreateAIJ(PetscObjectComm((PetscObject)da),Ng,PETSC_DECIDE,PETSC_DECIDE,Ntotal,Nint+Nsurf,NULL,Nint,NULL,P);
CHKERRQ(ierr);
} else {
ierr = MatZeroEntries(*P);
CHKERRQ(ierr);
}
ierr = MatSetOption(*P,MAT_ROW_ORIENTED,PETSC_FALSE);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nint,IIint,6,gl,xint,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nsurf,IIsurf,6,gl,xsurf,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = PetscFree2(IIint,IIsurf);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
{
Vec x,y;
PetscScalar *yy;
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),Ng,PETSC_DETERMINE,&y);
CHKERRQ(ierr);
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),PETSC_DETERMINE,Ntotal,&x);
CHKERRQ(ierr);
ierr = VecSet(x,1.0);
CHKERRQ(ierr);
ierr = MatMult(*P,x,y);
CHKERRQ(ierr);
ierr = VecGetArray(y,&yy);
CHKERRQ(ierr);
for (i=0; i<Ng; i++) {
if (PetscAbsScalar(yy[i]-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong p interpolation at i %D value %g",i,(double)PetscAbsScalar(yy[i]));
}
ierr = VecRestoreArray(y,&yy);
CHKERRQ(ierr);
ierr = VecDestroy(x);
CHKERRQ(ierr);
ierr = VecDestroy(y);
CHKERRQ(ierr);
}
#endif
ierr = MatDestroy(&Aii);
CHKERRQ(ierr);
ierr = MatDestroy(&Ais);
CHKERRQ(ierr);
ierr = MatDestroy(&Asi);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = ISDestroy(&is);
CHKERRQ(ierr);
ierr = ISDestroy(&isint);
CHKERRQ(ierr);
ierr = ISDestroy(&issurf);
CHKERRQ(ierr);
ierr = MatDestroy(&Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&Xsurf);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv) {
Mat A,B;
PetscInt i,j,column;
PetscInt *di,*dj,*oi,*oj;
PetscScalar *oa,*da,value;
PetscRandom rctx;
PetscErrorCode ierr;
PetscBool equal;
Mat_SeqAIJ *daij,*oaij;
Mat_MPIAIJ *Ampiaij;
PetscMPIInt size,rank;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size == 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_OUTOFRANGE,"Must run with 2 or more processes");CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Create a mpiaij matrix for checking */
ierr = MatCreateAIJ(PETSC_COMM_WORLD,5,5,PETSC_DECIDE,PETSC_DECIDE,0,PETSC_NULL,0,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
for (i=5*rank; i<5*rank+5; i++) {
for (j=0; j<5*size; j++){
ierr = PetscRandomGetValue(rctx,&value);CHKERRQ(ierr);
column = (PetscInt) (5*size*PetscRealPart(value));
ierr = PetscRandomGetValue(rctx,&value);CHKERRQ(ierr);
ierr = MatSetValues(A,1,&i,1,&column,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
Ampiaij = (Mat_MPIAIJ*) A->data;
daij = (Mat_SeqAIJ*) Ampiaij->A->data;
oaij = (Mat_SeqAIJ*) Ampiaij->B->data;
di = daij->i;
dj = daij->j;
da = daij->a;
oi = oaij->i;
oa = oaij->a;
ierr = PetscMalloc(oi[5]*sizeof(PetscInt),&oj);CHKERRQ(ierr);
ierr = PetscMemcpy(oj,oaij->j,oi[5]*sizeof(PetscInt));CHKERRQ(ierr);
/* modify the column entries in the non-diagonal portion back to global numbering */
for (i=0; i<oi[5]; i++) {
oj[i] = Ampiaij->garray[oj[i]];
}
ierr = MatCreateMPIAIJWithSplitArrays(PETSC_COMM_WORLD,5,5,PETSC_DETERMINE,PETSC_DETERMINE,di,dj,da,oi,oj,oa,&B);CHKERRQ(ierr);
ierr = MatSetUp(B);CHKERRQ(ierr);
ierr = MatEqual(A,B,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Likely a bug in MatCreateMPIAIJWithSplitArrays()");
/* Free spaces */
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscFree(oj);CHKERRQ(ierr);
ierr = PetscFinalize();
return(0);
}
int main(int argc,char **args)
{
MatType mtype = MATMPIAIJ; /* matrix format */
Mat A,B; /* matrix */
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
PetscBool flg,viewMats,viewIS,viewVecs;
PetscInt ierr,*nlocal,m,n;
PetscMPIInt rank,size;
MatPartitioning part;
IS is,isn;
Vec xin, xout;
VecScatter scat;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_mats", &viewMats);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_is", &viewIS);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_vecs", &viewVecs);CHKERRQ(ierr);
/*
Determine file from which we read the matrix
*/
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
/*
Open binary file. Note that we use FILE_MODE_READ to indicate
reading from this file.
*/
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
/*
Load the matrix and vector; then destroy the viewer.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,mtype);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&xin);CHKERRQ(ierr);
ierr = VecLoad(xin,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Original matrix:\n");
ierr = MatView(A,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
if (viewVecs) {
if (!rank) printf("Original vector:\n");
ierr = VecView(xin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Partition the graph of the matrix */
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* get new processor owner number of each vertex */
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS1 - new processor ownership:\n");
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* get new global number of each old global number */
ierr = ISPartitioningToNumbering(is,&isn);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS2 - new global numbering:\n");
ierr = ISView(isn,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* get number of new vertices for each processor */
ierr = PetscMalloc(size*sizeof(PetscInt),&nlocal);CHKERRQ(ierr);
ierr = ISPartitioningCount(is,size,nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
/* get old global number of each new global number */
ierr = ISInvertPermutation(isn,nlocal[rank],&is);CHKERRQ(ierr);
ierr = PetscFree(nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&isn);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS3=inv(IS2) - old global number of each new global number:\n");
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* move the matrix rows to the new processes they have been assigned to by the permutation */
ierr = ISSort(is);CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,is,is,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
/* move the vector rows to the new processes they have been assigned to */
ierr = MatGetLocalSize(B,&m,&n);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,m,PETSC_DECIDE,&xout);CHKERRQ(ierr);
ierr = VecScatterCreate(xin,is,xout,NULL,&scat);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,xin,xout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,xin,xout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Partitioned matrix:\n");
ierr = MatView(B,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
if (viewVecs) {
if (!rank) printf("Mapped vector:\n");
ierr = VecView(xout,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
{
PetscInt rstart,i,*nzd,*nzo,nzl,nzmax = 0,*ncols,nrow,j;
Mat J;
const PetscInt *cols;
const PetscScalar *vals;
PetscScalar *nvals;
ierr = MatGetOwnershipRange(B,&rstart,NULL);CHKERRQ(ierr);
ierr = PetscMalloc(2*m*sizeof(PetscInt),&nzd);CHKERRQ(ierr);
ierr = PetscMemzero(nzd,2*m*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMalloc(2*m*sizeof(PetscInt),&nzo);CHKERRQ(ierr);
ierr = PetscMemzero(nzo,2*m*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = MatGetRow(B,i+rstart,&nzl,&cols,NULL);CHKERRQ(ierr);
for (j=0; j<nzl; j++) {
if (cols[j] >= rstart && cols[j] < rstart+n) {
nzd[2*i] += 2;
nzd[2*i+1] += 2;
} else {
nzo[2*i] += 2;
nzo[2*i+1] += 2;
}
}
nzmax = PetscMax(nzmax,nzd[2*i]+nzo[2*i]);
ierr = MatRestoreRow(B,i+rstart,&nzl,&cols,NULL);CHKERRQ(ierr);
}
ierr = MatCreateAIJ(PETSC_COMM_WORLD,2*m,2*m,PETSC_DECIDE,PETSC_DECIDE,0,nzd,0,nzo,&J);CHKERRQ(ierr);
ierr = PetscInfo(0,"Created empty Jacobian matrix\n");CHKERRQ(ierr);
ierr = PetscFree(nzd);CHKERRQ(ierr);
ierr = PetscFree(nzo);CHKERRQ(ierr);
ierr = PetscMalloc2(nzmax,PetscInt,&ncols,nzmax,PetscScalar,&nvals);CHKERRQ(ierr);
ierr = PetscMemzero(nvals,nzmax*sizeof(PetscScalar));CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = MatGetRow(B,i+rstart,&nzl,&cols,&vals);CHKERRQ(ierr);
for (j=0; j<nzl; j++) {
ncols[2*j] = 2*cols[j];
ncols[2*j+1] = 2*cols[j]+1;
}
nrow = 2*(i+rstart);
ierr = MatSetValues(J,1,&nrow,2*nzl,ncols,nvals,INSERT_VALUES);CHKERRQ(ierr);
nrow = 2*(i+rstart) + 1;
ierr = MatSetValues(J,1,&nrow,2*nzl,ncols,nvals,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(B,i+rstart,&nzl,&cols,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Jacobian matrix structure:\n");
ierr = MatView(J,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = PetscFree2(ncols,nvals);CHKERRQ(ierr);
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecDestroy(&xin);CHKERRQ(ierr);
ierr = VecDestroy(&xout);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
