int main(int argc,char **argv)
{
PetscInt n = 5;
PetscErrorCode ierr;
PetscMPIInt rank,size;
IS ispetsc,isapp;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* create the index sets */
ierr = ISCreateStride(PETSC_COMM_WORLD,n,rank,size,&ispetsc);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,n,n*rank,1,&isapp);CHKERRQ(ierr);
/* create the application ordering */
ierr = AOCreateBasicIS(isapp,ispetsc,&ao);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(ispetsc,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(isapp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplicationIS(ao,ispetsc);CHKERRQ(ierr);
ierr = ISView(isapp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(ispetsc,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&ispetsc);CHKERRQ(ierr);
ierr = ISDestroy(&isapp);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
static PetscErrorCode ISView_Block(IS is, PetscViewer viewer)
{
IS_Block *sub = (IS_Block*)is->data;
PetscErrorCode ierr;
PetscInt i,bs,n,*idx = sub->idx;
PetscBool iascii;
PetscFunctionBegin;
ierr = PetscLayoutGetBlockSize(is->map, &bs);CHKERRQ(ierr);
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
n /= bs;
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
if (iascii) {
PetscViewerFormat fmt;
ierr = PetscViewerGetFormat(viewer,&fmt);CHKERRQ(ierr);
if (fmt == PETSC_VIEWER_ASCII_MATLAB) {
IS ist;
const char *name;
const PetscInt *idx;
PetscInt n;
ierr = PetscObjectGetName((PetscObject)is,&name);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idx);CHKERRQ(ierr);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)is),n,idx,PETSC_USE_POINTER,&ist);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)ist,name);CHKERRQ(ierr);
ierr = ISView(ist,viewer);CHKERRQ(ierr);
ierr = ISDestroy(&ist);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idx);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
if (is->isperm) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Block Index set is permutation\n");CHKERRQ(ierr);
}
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Block size %D\n",bs);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Number of block indices in set %D\n",n);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"The first indices of each block are\n");CHKERRQ(ierr);
for (i=0; i<n; i++) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Block %D Index %D\n",i,idx[i]);CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n,first,step;
IS set;
const PetscInt *indices;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
n = 10;
first = 3;
step = 2;
/*
Create stride index set, starting at 3 with a stride of 2
Note each processor is generating its own index set
(in this case they are all identical)
*/
ierr = ISCreateStride(PETSC_COMM_SELF,n,first,step,&set);CHKERRQ(ierr);
ierr = ISView(set,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Extract indices from set.
*/
ierr = ISGetIndices(set,&indices);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Printing indices directly\n");CHKERRQ(ierr);
for (i=0; i<n; i++) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"%D\n",indices[i]);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(set,&indices);CHKERRQ(ierr);
/*
Determine information on stride
*/
ierr = ISStrideGetInfo(set,&first,&step);CHKERRQ(ierr);
if (first != 3 || step != 2) SETERRQ(PETSC_COMM_SELF,1,"Stride info not correct!\n");
ierr = ISDestroy(&set);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat mesh,dual;
PetscErrorCode ierr;
PetscInt Nvertices = 6; /* total number of vertices */
PetscInt ncells = 2; /* number cells on this process */
PetscInt *ii,*jj;
PetscMPIInt size,rank;
MatPartitioning part;
IS is;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(MPI_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This example is for exactly two processes");
ierr = MPI_Comm_rank(MPI_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscMalloc(3*sizeof(PetscInt),&ii);CHKERRQ(ierr);
ierr = PetscMalloc(6*sizeof(PetscInt),&jj);CHKERRQ(ierr);
ii[0] = 0; ii[1] = 3; ii[2] = 6;
if (!rank) {
jj[0] = 0; jj[1] = 1; jj[2] = 2; jj[3] = 1; jj[4] = 3; jj[5] = 2;
} else {
jj[0] = 1; jj[1] = 4; jj[2] = 5; jj[3] = 1; jj[4] = 5; jj[5] = 3;
}
ierr = MatCreateMPIAdj(MPI_COMM_WORLD,ncells,Nvertices,ii,jj,NULL,&mesh);CHKERRQ(ierr);
ierr = MatMeshToCellGraph(mesh,2,&dual);CHKERRQ(ierr);
ierr = MatView(dual,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatPartitioningCreate(MPI_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,dual);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&mesh);CHKERRQ(ierr);
ierr = MatDestroy(&dual);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
TEST(la, inversion_matrix) {
srand(time(NULL));
int first_idx=0, size=0, submat_blocks=6;
IS set;
// vytvorit rozdeleni bloku na procesory ve tvaru "part" (tj. indexy prvnich radku na procesorech)
int np, rank;
double block_size;
int min_idx, max_idx;
MPI_Comm_size(PETSC_COMM_WORLD, &np);
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
block_size = (double)submat_blocks / (double)np;
min_idx = (int) ( round(block_size * rank) );
max_idx = (int) ( round(block_size * (rank + 1)) );
for (int i = 0; i < min_idx; i++) {
first_idx += rows[i];
}
for (int i = min_idx; i < max_idx; i++) {
size += rows[i];
}
// volat s lokalni velkosti = pocet radku na lokalnim proc.
LinSys * lin_sys = new LinSys_MPIAIJ(size + max_idx - min_idx);
lin_sys->set_symmetric();
lin_sys->start_allocation();
fill_matrix( lin_sys, min_idx, max_idx ); // preallocate matrix
lin_sys->start_add_assembly();
fill_matrix( lin_sys, min_idx, max_idx ); // fill matrix
lin_sys->finalize();
MatView(lin_sys->get_matrix(),PETSC_VIEWER_STDOUT_WORLD);
ISCreateStride(PETSC_COMM_WORLD, size, first_idx + min_idx, 1, &set); // kazdy proc. lokalni cast indexsetu viz. schur.cc line 386
ISView(set, PETSC_VIEWER_STDOUT_WORLD);
SchurComplement schurComplement(lin_sys, set, 6);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n,*indices;
PetscInt rank,size;
IS is,newis;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/*
Create IS
*/
n = 4 + rank;
ierr = PetscMalloc(n*sizeof(PetscInt),&indices);CHKERRQ(ierr);
for (i=0; i<n; i++) {
indices[i] = rank + i;
}
ierr = ISCreateGeneral(PETSC_COMM_WORLD,n,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
/*
Stick them together from all processors
*/
ierr = ISAllGather(is,&newis);CHKERRQ(ierr);
if (!rank) {
ierr = ISView(newis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = ISDestroy(&newis);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
MatPartitioningApply - Gets a partitioning for a matrix.
Collective on Mat
Input Parameters:
. matp - the matrix partitioning object
Output Parameters:
. partitioning - the partitioning. For each local node this tells the processor
number that that node is assigned to.
Options Database Keys:
To specify the partitioning through the options database, use one of
the following
$ -mat_partitioning_type parmetis, -mat_partitioning current
To see the partitioning result
$ -mat_partitioning_view
Level: beginner
The user can define additional partitionings; see MatPartitioningRegister().
.keywords: matrix, get, partitioning
.seealso: MatPartitioningRegister(), MatPartitioningCreate(),
MatPartitioningDestroy(), MatPartitioningSetAdjacency(), ISPartitioningToNumbering(),
ISPartitioningCount()
@*/
PetscErrorCode MatPartitioningApply(MatPartitioning matp,IS *partitioning)
{
PetscErrorCode ierr;
PetscBool flag = PETSC_FALSE;
PetscFunctionBegin;
PetscValidHeaderSpecific(matp,MAT_PARTITIONING_CLASSID,1);
PetscValidPointer(partitioning,2);
if (!matp->adj->assembled) SETERRQ(PetscObjectComm((PetscObject)matp),PETSC_ERR_ARG_WRONGSTATE,"Not for unassembled matrix");
if (matp->adj->factortype) SETERRQ(PetscObjectComm((PetscObject)matp),PETSC_ERR_ARG_WRONGSTATE,"Not for factored matrix");
if (!matp->ops->apply) SETERRQ(PetscObjectComm((PetscObject)matp),PETSC_ERR_ARG_WRONGSTATE,"Must set type with MatPartitioningSetFromOptions() or MatPartitioningSetType()");
ierr = PetscLogEventBegin(MAT_Partitioning,matp,0,0,0);CHKERRQ(ierr);
ierr = (*matp->ops->apply)(matp,partitioning);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_Partitioning,matp,0,0,0);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-mat_partitioning_view",&flag,NULL);CHKERRQ(ierr);
if (flag) {
PetscViewer viewer;
ierr = PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)matp),&viewer);CHKERRQ(ierr);
ierr = MatPartitioningView(matp,viewer);CHKERRQ(ierr);
ierr = ISView(*partitioning,viewer);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@C
ISColoringView - Views a coloring context.
Collective on ISColoring
Input Parameters:
+ iscoloring - the coloring context
- viewer - the viewer
Level: advanced
.seealso: ISColoringDestroy(), ISColoringGetIS(), MatColoring
@*/
PetscErrorCode ISColoringView(ISColoring iscoloring,PetscViewer viewer)
{
PetscInt i;
PetscErrorCode ierr;
PetscBool iascii;
IS *is;
PetscFunctionBegin;
PetscValidPointer(iscoloring,1);
if (!viewer) {
ierr = PetscViewerASCIIGetStdout(iscoloring->comm,&viewer);CHKERRQ(ierr);
}
PetscValidHeaderSpecific(viewer,PETSC_VIEWER_CLASSID,2);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
if (iascii) {
MPI_Comm comm;
PetscMPIInt size,rank;
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"ISColoring Object: %d MPI processes\n",size);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr);
} else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
ierr = ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);CHKERRQ(ierr);
for (i=0; i<iscoloring->n; i++) {
ierr = ISView(iscoloring->is[i],viewer);CHKERRQ(ierr);
}
ierr = ISColoringRestoreIS(iscoloring,&is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat C,Cperm;
PetscInt i,j,m = 5,n = 5,Ii,J,ncols;
PetscErrorCode ierr;
PetscScalar v;
PetscMPIInt size;
IS rperm,cperm,icperm;
const PetscInt *rperm_ptr,*cperm_ptr,*cols;
const PetscScalar *vals;
PetscBool TestMyorder=PETSC_FALSE;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);
CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!");
/* create the matrix for the five point stencil, YET AGAIN */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,m*n,m*n,5,NULL,&C);
ierr = MatSetUp(C);
CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=0; j<n; j++) {
v = -1.0;
Ii = j + n*i;
if (i>0) {
J = Ii - n;
ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<m-1) {
J = Ii + n;
ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n-1) {
J = Ii + 1;
ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);
CHKERRQ(ierr);
}
v = 4.0;
ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatGetOrdering(C,MATORDERINGND,&rperm,&cperm);
CHKERRQ(ierr);
ierr = ISView(rperm,PETSC_VIEWER_STDOUT_SELF);
CHKERRQ(ierr);
ierr = ISDestroy(&rperm);
CHKERRQ(ierr);
ierr = ISDestroy(&cperm);
CHKERRQ(ierr);
ierr = MatGetOrdering(C,MATORDERINGRCM,&rperm,&cperm);
CHKERRQ(ierr);
ierr = ISView(rperm,PETSC_VIEWER_STDOUT_SELF);
CHKERRQ(ierr);
ierr = ISDestroy(&rperm);
CHKERRQ(ierr);
ierr = ISDestroy(&cperm);
CHKERRQ(ierr);
ierr = MatGetOrdering(C,MATORDERINGQMD,&rperm,&cperm);
CHKERRQ(ierr);
ierr = ISView(rperm,PETSC_VIEWER_STDOUT_SELF);
CHKERRQ(ierr);
ierr = ISDestroy(&rperm);
CHKERRQ(ierr);
ierr = ISDestroy(&cperm);
CHKERRQ(ierr);
/* create Cperm = rperm*C*icperm */
ierr = PetscOptionsGetBool(NULL,"-testmyordering",&TestMyorder,NULL);
CHKERRQ(ierr);
if (TestMyorder) {
ierr = MatGetOrdering_myordering(C,MATORDERINGQMD,&rperm,&cperm);
CHKERRQ(ierr);
printf("myordering's rperm:\n");
ierr = ISView(rperm,PETSC_VIEWER_STDOUT_SELF);
CHKERRQ(ierr);
ierr = ISInvertPermutation(cperm,PETSC_DECIDE,&icperm);
CHKERRQ(ierr);
ierr = ISGetIndices(rperm,&rperm_ptr);
CHKERRQ(ierr);
ierr = ISGetIndices(icperm,&cperm_ptr);
CHKERRQ(ierr);
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,m*n,m*n,5,NULL,&Cperm);
CHKERRQ(ierr);
for (i=0; i<m*n; i++) {
ierr = MatGetRow(C,rperm_ptr[i],&ncols,&cols,&vals);
CHKERRQ(ierr);
for (j=0; j<ncols; j++) {
/* printf(" (%d %d %g)\n",i,cperm_ptr[cols[j]],vals[j]); */
ierr = MatSetValues(Cperm,1,&i,1,&cperm_ptr[cols[j]],&vals[j],INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(Cperm,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(Cperm,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = ISRestoreIndices(rperm,&rperm_ptr);
CHKERRQ(ierr);
ierr = ISRestoreIndices(icperm,&cperm_ptr);
CHKERRQ(ierr);
ierr = ISDestroy(&rperm);
CHKERRQ(ierr);
ierr = ISDestroy(&cperm);
CHKERRQ(ierr);
ierr = ISDestroy(&icperm);
CHKERRQ(ierr);
ierr = MatDestroy(&Cperm);
CHKERRQ(ierr);
}
ierr = MatDestroy(&C);
CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt *indices,n;
const PetscInt *nindices;
PetscMPIInt rank;
IS is;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/*
Create an index set with 5 entries. Each processor creates
its own index set with its own list of integers.
*/
ierr = PetscMalloc1(5,&indices);CHKERRQ(ierr);
indices[0] = rank + 1;
indices[1] = rank + 2;
indices[2] = rank + 3;
indices[3] = rank + 4;
indices[4] = rank + 5;
ierr = ISCreateGeneral(PETSC_COMM_SELF,5,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
/*
Note that ISCreateGeneral() has made a copy of the indices
so we may (and generally should) free indices[]
*/
ierr = PetscFree(indices);CHKERRQ(ierr);
/*
Print the index set to stdout
*/
ierr = ISView(is,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Get the number of indices in the set
*/
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
/*
Get the indices in the index set
*/
ierr = ISGetIndices(is,&nindices);CHKERRQ(ierr);
/*
Now any code that needs access to the list of integers
has access to it here through indices[].
*/
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] First index %D\n",rank,nindices[0]);CHKERRQ(ierr);
/*
Once we no longer need access to the indices they should
returned to the system
*/
ierr = ISRestoreIndices(is,&nindices);CHKERRQ(ierr);
/*
One should destroy any PETSc object once one is completely
done with it.
*/
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PETSC_EXTERN void PETSC_STDCALL isview_(IS *is,PetscViewer *vin,PetscErrorCode *ierr)
{
PetscViewer v;
PetscPatchDefaultViewers_Fortran(vin,v);
*ierr = ISView(*is,v);
}
int main(int argc,char **args)
{
Mat C,Caij;
PetscInt i,j,m = 5,n,nrows,ncols;
const PetscInt *rows,*cols;
IS isrows,iscols;
PetscErrorCode ierr;
PetscBool flg,Test_MatMatMult=PETSC_FALSE,mats_view=PETSC_FALSE;
PetscScalar *v;
PetscMPIInt rank,size;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-mats_view",&mats_view);CHKERRQ(ierr);
/* Get local block or element size*/
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
n = m;
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(C,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-row_oriented",&flg);CHKERRQ(ierr);
if (flg) {ierr = MatSetOption(C,MAT_ROW_ORIENTED,PETSC_TRUE);CHKERRQ(ierr);}
ierr = MatGetOwnershipIS(C,&isrows,&iscols);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-Cexp_view_ownership",&flg);CHKERRQ(ierr);
if (flg) { /* View ownership of explicit C */
IS tmp;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Ownership of explicit C:\n");CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row index set:\n");CHKERRQ(ierr);
ierr = ISOnComm(isrows,PETSC_COMM_WORLD,PETSC_USE_POINTER,&tmp);CHKERRQ(ierr);
ierr = ISView(tmp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Column index set:\n");CHKERRQ(ierr);
ierr = ISOnComm(iscols,PETSC_COMM_WORLD,PETSC_USE_POINTER,&tmp);CHKERRQ(ierr);
ierr = ISView(tmp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Set local matrix entries */
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc(nrows*ncols*sizeof(*v),&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
/*v[i*ncols+j] = (PetscReal)(rank);*/
v[i*ncols+j] = (PetscReal)(rank*10000+100*rows[i]+cols[j]);
}
}
ierr = MatSetValues(C,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatView() */
if (mats_view) {
ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Set unowned matrix entries - add subdiagonals and diagonals from proc[0] */
if (!rank) {
PetscInt M,N,cols[2];
ierr = MatGetSize(C,&M,&N);CHKERRQ(ierr);
for (i=0; i<M; i++) {
cols[0] = i; v[0] = i + 0.5;
cols[1] = i-1; v[1] = 0.5;
if (i) {
ierr = MatSetValues(C,1,&i,2,cols,v,ADD_VALUES);CHKERRQ(ierr);
} else {
ierr = MatSetValues(C,1,&i,1,&i,v,ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatMult() */
ierr = MatComputeExplicitOperator(C,&Caij);CHKERRQ(ierr);
ierr = MatMultEqual(C,Caij,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"C != Caij. MatMultEqual() fails");
ierr = MatMultTransposeEqual(C,Caij,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"C != Caij. MatMultTransposeEqual() fails");
/* Test MatMultAdd() and MatMultTransposeAddEqual() */
ierr = MatMultAddEqual(C,Caij,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"C != Caij. MatMultAddEqual() fails");
ierr = MatMultTransposeAddEqual(C,Caij,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"C != Caij. MatMultTransposeAddEqual() fails");
/* Test MatMatMult() */
ierr = PetscOptionsHasName(NULL,"-test_matmatmult",&Test_MatMatMult);CHKERRQ(ierr);
if (Test_MatMatMult) {
Mat CCelem,CCaij;
ierr = MatMatMult(C,C,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&CCelem);CHKERRQ(ierr);
ierr = MatMatMult(Caij,Caij,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&CCaij);CHKERRQ(ierr);
ierr = MatMultEqual(CCelem,CCaij,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"CCelem != CCaij. MatMatMult() fails");
ierr = MatDestroy(&CCaij);CHKERRQ(ierr);
ierr = MatDestroy(&CCelem);CHKERRQ(ierr);
}
ierr = PetscFree(v);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&Caij);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode TestMatZeroRows(Mat A, Mat Afull, PetscBool squaretest, IS is, PetscScalar diag)
{
Mat B,Bcheck,B2 = NULL,lB;
Vec x = NULL, b = NULL, b2 = NULL;
ISLocalToGlobalMapping l2gr,l2gc;
PetscReal error;
char diagstr[16];
const PetscInt *idxs;
PetscInt rst,ren,i,n,N,d;
PetscMPIInt rank;
PetscBool miss,haszerorows;
PetscErrorCode ierr;
PetscFunctionBeginUser;
if (diag == 0.) {
ierr = PetscStrcpy(diagstr,"zero");CHKERRQ(ierr);
} else {
ierr = PetscStrcpy(diagstr,"nonzero");CHKERRQ(ierr);
}
ierr = ISView(is,NULL);CHKERRQ(ierr);
ierr = MatGetLocalToGlobalMapping(A,&l2gr,&l2gc);CHKERRQ(ierr);
/* tests MatDuplicate and MatCopy */
if (diag == 0.) {
ierr = MatDuplicate(A,MAT_COPY_VALUES,&B);CHKERRQ(ierr);
} else {
ierr = MatDuplicate(A,MAT_DO_NOT_COPY_VALUES,&B);CHKERRQ(ierr);
ierr = MatCopy(A,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = MatISGetLocalMat(B,&lB);CHKERRQ(ierr);
ierr = MatHasOperation(lB,MATOP_ZERO_ROWS,&haszerorows);CHKERRQ(ierr);
if (squaretest && haszerorows) {
ierr = MatCreateVecs(B,&x,&b);CHKERRQ(ierr);
ierr = MatDuplicate(B,MAT_COPY_VALUES,&B2);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(b,l2gr);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(x,l2gc);CHKERRQ(ierr);
ierr = VecSetRandom(x,NULL);CHKERRQ(ierr);
ierr = VecSetRandom(b,NULL);CHKERRQ(ierr);
/* mimic b[is] = x[is] */
ierr = VecDuplicate(b,&b2);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(b2,l2gr);CHKERRQ(ierr);
ierr = VecCopy(b,b2);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idxs);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
for (i=0;i<n;i++) {
if (0 <= idxs[i] && idxs[i] < N) {
ierr = VecSetValue(b2,idxs[i],diag,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValue(x,idxs[i],1.,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = VecAssemblyBegin(b2);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b2);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idxs);CHKERRQ(ierr);
/* test ZeroRows on MATIS */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsIS(B,is,diag,x,b);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRowsColumns (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsIS(B2,is,diag,NULL,NULL);CHKERRQ(ierr);
} else if (haszerorows) {
/* test ZeroRows on MATIS */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsIS(B,is,diag,NULL,NULL);CHKERRQ(ierr);
b = b2 = x = NULL;
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Skipping MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
b = b2 = x = NULL;
}
if (squaretest && haszerorows) {
ierr = VecAXPY(b2,-1.,b);CHKERRQ(ierr);
ierr = VecNorm(b2,NORM_INFINITY,&error);CHKERRQ(ierr);
if (error > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"ERROR IN ZEROROWS ON B %g (diag %s)",error,diagstr);
}
/* test MatMissingDiagonal */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatMissingDiagonal\n");CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MatMissingDiagonal(B,&miss,&d);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(B,&rst,&ren);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD, "[%d] [%D,%D) Missing %d, row %D (diag %s)\n",rank,rst,ren,(int)miss,d,diagstr);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&b2);CHKERRQ(ierr);
/* check the result of ZeroRows with that from MPIAIJ routines
assuming that MatConvert_IS_XAIJ and MatZeroRows_MPIAIJ work fine */
if (haszerorows) {
ierr = MatDuplicate(Afull,MAT_COPY_VALUES,&Bcheck);CHKERRQ(ierr);
ierr = MatSetOption(Bcheck,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatZeroRowsIS(Bcheck,is,diag,NULL,NULL);CHKERRQ(ierr);
ierr = CheckMat(B,Bcheck,PETSC_FALSE,"Zerorows");CHKERRQ(ierr);
ierr = MatDestroy(&Bcheck);CHKERRQ(ierr);
}
ierr = MatDestroy(&B);CHKERRQ(ierr);
if (B2) { /* test MatZeroRowsColumns */
ierr = MatDuplicate(Afull,MAT_COPY_VALUES,&B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsIS(B,is,diag,NULL,NULL);CHKERRQ(ierr);
ierr = CheckMat(B2,B,PETSC_FALSE,"MatZeroRowsColumns");CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&B2);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
Mat A;
MatPartitioning part;
IS is;
PetscInt r,N = 10, start, end;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char*) 0, help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL, "-N", &N, NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD, &A);CHKERRQ(ierr);
ierr = MatSetSizes(A, PETSC_DECIDE, PETSC_DECIDE, N, N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A, 3, NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A, 3, NULL, 2, NULL);CHKERRQ(ierr);
/* Create a linear mesh */
ierr = MatGetOwnershipRange(A, &start, &end);CHKERRQ(ierr);
for (r = start; r < end; ++r) {
if (r == 0) {
PetscInt cols[2];
PetscScalar vals[2];
cols[0] = r; cols[1] = r+1;
vals[0] = 1.0; vals[1] = 1.0;
ierr = MatSetValues(A, 1, &r, 2, cols, vals, INSERT_VALUES);CHKERRQ(ierr);
} else if (r == N-1) {
PetscInt cols[2];
PetscScalar vals[2];
cols[0] = r-1; cols[1] = r;
vals[0] = 1.0; vals[1] = 1.0;
ierr = MatSetValues(A, 1, &r, 2, cols, vals, INSERT_VALUES);CHKERRQ(ierr);
} else {
PetscInt cols[3];
PetscScalar vals[3];
cols[0] = r-1; cols[1] = r; cols[2] = r+1;
vals[0] = 1.0; vals[1] = 1.0; vals[2] = 1.0;
ierr = MatSetValues(A, 1, &r, 3, cols, vals, INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatPartitioningCreate(PETSC_COMM_WORLD, &part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part, A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/*ierr = MatPartitioningSetVertexWeights(part, const PetscInt weights[]);CHKERRQ(ierr);*/
/*ierr = MatPartitioningSetPartitionWeights(part,const PetscReal weights[]);CHKERRQ(ierr);*/
ierr = MatPartitioningApply(part, &is);CHKERRQ(ierr);
ierr = ISView(is, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
Mat mat,B;
PetscErrorCode ierr;
PetscInt i,j;
PetscScalar v;
IS isrow,iscol;
PetscViewer viewer;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* ------- Assemble matrix, --------- */
ierr = MatCreate(PETSC_COMM_WORLD,&mat);CHKERRQ(ierr);
ierr = MatSetSizes(mat,PETSC_DECIDE,PETSC_DECIDE,4,4);CHKERRQ(ierr);
ierr = MatSetFromOptions(mat);CHKERRQ(ierr);
ierr = MatSetUp(mat);CHKERRQ(ierr);
/* set anti-diagonal of matrix */
v = 1.0;
i = 0; j = 3;
ierr = MatSetValues(mat,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
v = 2.0;
i = 1; j = 2;
ierr = MatSetValues(mat,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
v = 3.0;
i = 2; j = 1;
ierr = MatSetValues(mat,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
v = 4.0;
i = 3; j = 0;
ierr = MatSetValues(mat,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_DENSE);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix\n");CHKERRQ(ierr);
ierr = MatView(mat,viewer);CHKERRQ(ierr);
ierr = MatGetOrdering(mat,MATORDERINGNATURAL,&isrow,&iscol);CHKERRQ(ierr);
ierr = MatPermute(mat,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix permuted by identity\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(mat,1.e-8,isrow,iscol);CHKERRQ(ierr);
ierr = MatPermute(mat,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix permuted by identity + NonzeroDiagonal()\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Column permutation\n");CHKERRQ(ierr);
ierr = ISView(iscol,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatGetOrdering(mat,MATORDERINGND,&isrow,&iscol);CHKERRQ(ierr);
ierr = MatPermute(mat,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix permuted by ND\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"ND row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"ND column permutation\n");CHKERRQ(ierr);
ierr = ISView(iscol,viewer);CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(mat,1.e-8,isrow,iscol);CHKERRQ(ierr);
ierr = MatPermute(mat,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix permuted by ND + NonzeroDiagonal()\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"ND + NonzeroDiagonal() row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"ND + NonzeroDiagonal() column permutation\n");CHKERRQ(ierr);
ierr = ISView(iscol,viewer);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatGetOrdering(mat,MATORDERINGRCM,&isrow,&iscol);CHKERRQ(ierr);
ierr = MatPermute(mat,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix permuted by RCM\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"RCM row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"RCM column permutation\n");CHKERRQ(ierr);
ierr = ISView(iscol,viewer);CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(mat,1.e-8,isrow,iscol);CHKERRQ(ierr);
ierr = MatPermute(mat,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix permuted by RCM + NonzeroDiagonal()\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"RCM + NonzeroDiagonal() row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"RCM + NonzeroDiagonal() column permutation\n");CHKERRQ(ierr);
ierr = ISView(iscol,viewer);CHKERRQ(ierr);
ierr = MatLUFactor(mat,isrow,iscol,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Factored matrix permuted by RCM + NonzeroDiagonal()\n");CHKERRQ(ierr);
ierr = MatView(mat,viewer);CHKERRQ(ierr);
/* Free data structures */
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode private_ISView_Swarm_XDMF(IS is,PetscViewer viewer)
{
long int *bytes = NULL;
PetscContainer container = NULL;
const char *viewername;
char datafile[PETSC_MAX_PATH_LEN];
PetscViewer fviewer;
PetscInt N,bs;
const char *vecname;
char fieldname[PETSC_MAX_PATH_LEN];
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)viewer,"XDMFViewerContext",(PetscObject*)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void**)&bytes);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"Valid to find attached data XDMFViewerContext");
ierr = PetscViewerFileGetName(viewer,&viewername);CHKERRQ(ierr);
ierr = private_CreateDataFileNameXDMF(viewername,datafile);CHKERRQ(ierr);
/* re-open a sub-viewer for all data fields */
/* name is viewer.name + "_swarm_fields.pbin" */
ierr = PetscViewerCreate(PetscObjectComm((PetscObject)viewer),&fviewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(fviewer,PETSCVIEWERBINARY);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipHeader(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipInfo(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerFileSetMode(fviewer,FILE_MODE_APPEND);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(fviewer,datafile);CHKERRQ(ierr);
ierr = ISGetSize(is,&N);CHKERRQ(ierr);
ierr = ISGetBlockSize(is,&bs);CHKERRQ(ierr);
N = N/bs;
ierr = PetscObjectGetName((PetscObject)is,&vecname);CHKERRQ(ierr);
if (!vecname) {
ierr = PetscSNPrintf(fieldname,PETSC_MAX_PATH_LEN-1,"swarmfield_%D",((PetscObject)is)->tag);CHKERRQ(ierr);
} else {
ierr = PetscSNPrintf(fieldname,PETSC_MAX_PATH_LEN-1,"%s",vecname);CHKERRQ(ierr);
}
/* write data header */
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"<Attribute Center=\"Node\" Name=\"%s\" Type=\"None\">\n",fieldname);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
if (bs == 1) {
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Int\" Precision=\"4\" Dimensions=\"%D\" Seek=\"%D\">\n",N,bytes[0]);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Int\" Precision=\"4\" Dimensions=\"%D %D\" Seek=\"%D\">\n",N,bs,bytes[0]);CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s\n",datafile);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</DataItem>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</Attribute>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
/* write data */
ierr = ISView(is,fviewer);CHKERRQ(ierr);
bytes[0] += sizeof(PetscInt) * N * bs;
ierr = PetscViewerDestroy(&fviewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A, *S;
IS rowis[2], colis[2];
PetscInt n,N,i,j,k,l,nsub,Jlow[2] = {0,1}, *jlow, Jhigh[2] = {3,4}, *jhigh, row, col, *subindices, ncols;
const PetscInt *cols;
PetscScalar v;
PetscMPIInt rank, size, p, inversions, total_inversions;
PetscBool sort_rows, sort_cols, show_inversions;
PetscErrorCode ierr;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size>2) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG, "A uniprocessor or two-processor example only.\n");
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
if (size > 1) {
n = 8; N = 16;
} else {
n = 16; N = 16;
}
ierr = MatSetSizes(A,n,n,N,N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/* Don't care if the entries are set multiple times by different procs. */
for (i=0; i<4; ++i) {
for (j = 0; j<4; ++j) {
row = j*4+i;
v = -1.0;
if (i>0) {
col = row-1; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (i<3) {
col = row+1; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>0) {
col = row-4; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (j<3) {
col = row+4; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
v = 4.0;
ierr = MatSetValues(A,1,&row,1,&row,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Original matrix\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (size > 1) {
nsub = 1; /* one subdomain per rank */
}
else {
nsub = 2; /* both subdomains on rank 0 */
}
if (rank) {
jlow = Jlow+1; jhigh = Jhigh+1;
}
else {
jlow = Jlow; jhigh = Jhigh;
}
sort_rows = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-sort_rows", &sort_rows, NULL);CHKERRQ(ierr);
sort_cols = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-sort_cols", &sort_cols, NULL);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
ierr = PetscMalloc1(12, &subindices);CHKERRQ(ierr);
k = 0;
for (i = 0; i < 4; ++i) {
for (j = jlow[l]; j < jhigh[l]; ++j) {
subindices[k] = j*4+i;
k++;
}
}
ierr = ISCreateGeneral(PETSC_COMM_SELF, 12, subindices, PETSC_OWN_POINTER, rowis+l);CHKERRQ(ierr);
if ((sort_rows && !sort_cols) || (!sort_rows && sort_cols)) {
ierr = ISDuplicate(rowis[l],colis+l);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)rowis[l]);CHKERRQ(ierr);
colis[l] = rowis[l];
}
if (sort_rows) {
ierr = ISSort(rowis[l]);CHKERRQ(ierr);
}
if (sort_cols) {
ierr = ISSort(colis[l]);CHKERRQ(ierr);
}
}
ierr = PetscMalloc1(nsub, &S);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,nsub,rowis,colis,MAT_INITIAL_MATRIX, &S);CHKERRQ(ierr);
show_inversions = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-show_inversions", &show_inversions, NULL);CHKERRQ(ierr);
inversions = 0;
for (p = 0; p < size; ++p) {
if (p == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Number of subdomains: %D:\n", rank, size, nsub);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
PetscInt i0, i1;
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Subdomain row IS %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = ISView(rowis[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Subdomain col IS %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = ISView(colis[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Submatrix %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = MatView(S[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (show_inversions) {
ierr = MatGetOwnershipRange(S[l], &i0,&i1);CHKERRQ(ierr);
for (i = i0; i < i1; ++i) {
ierr = MatGetRow(S[l], i, &ncols, &cols, NULL);CHKERRQ(ierr);
for (j = 1; j < ncols; ++j) {
if (cols[j] < cols[j-1]) {
ierr = PetscPrintf(PETSC_COMM_SELF, "***Inversion in row %D: col[%D] = %D < %D = col[%D]\n", i, j, cols[j], cols[j-1], j-1);CHKERRQ(ierr);
inversions++;
}
}
ierr = MatRestoreRow(S[l], i, &ncols, &cols, NULL);CHKERRQ(ierr);
}
}
}
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
if (show_inversions) {
ierr = MPI_Reduce(&inversions,&total_inversions,1,MPIU_INT, MPIU_SUM,0,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "*Total inversions: %D\n", total_inversions);CHKERRQ(ierr);
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
ierr = MatDestroy(&(S[l]));CHKERRQ(ierr);
ierr = ISDestroy(&(rowis[l]));CHKERRQ(ierr);
ierr = ISDestroy(&(colis[l]));CHKERRQ(ierr);
}
ierr = PetscFree(S);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMView_AKKT(DM dm, PetscViewer v) {
DM_AKKT* kkt = (DM_AKKT*)(dm->data);
PetscErrorCode ierr;
PetscBool isascii;
PetscInt i, tab, vtab;
const char* name, *prefix;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)v, PETSCVIEWERASCII, &isascii); CHKERRQ(ierr);
if(!isascii) SETERRQ(((PetscObject)dm)->comm, PETSC_ERR_SUP, "No support for non-ASCII viewers");
ierr = PetscObjectGetTabLevel((PetscObject)dm, &tab); CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)dm, &name); CHKERRQ(ierr);
ierr = PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix); CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(v,PETSC_TRUE); CHKERRQ(ierr);
ierr = PetscViewerASCIIGetTab(v,&vtab); CHKERRQ(ierr);
ierr = PetscViewerASCIISetTab(v,tab); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(v, "DM Algebraic KKT, name: %s, prefix: %s\n", ((PetscObject)dm)->name, ((PetscObject)dm)->prefix); CHKERRQ(ierr);
if(kkt->dm) {
ierr = PetscViewerASCIIPrintf(v, "DM:\n"); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = DMView(kkt->dm,v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
if(kkt->Aff) {
ierr = PetscViewerASCIIPrintf(v, "Aff:\n"); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = MatView(kkt->Aff,v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
if(kkt->dname) {
ierr = PetscViewerASCIIPrintf(v, "Decomposition, name %s:\n"); CHKERRQ(ierr);
}
for(i = 0; i < 2; ++i) {
const char* label;
if(i == 0) {
label = "Primal";
}
else {
label = "Dual";
}
if(kkt->names[i]) {
ierr = PetscViewerASCIIPrintf(v, "%s, name %s:\n", label, kkt->names[i]); CHKERRQ(ierr);
}
if(kkt->isf[i]){
ierr = PetscViewerASCIIPrintf(v, "%s, IS:\n",label); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = ISView(kkt->isf[i],v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
if(kkt->dmf[i]){
ierr = PetscViewerASCIIPrintf(v, "%s, DM:\n", label); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = DMView(kkt->dmf[i],v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
}
if(kkt->Pfc) {
ierr = PetscViewerASCIIPrintf(v, "Prolongation:\n"); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = MatView(kkt->Pfc,v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
ierr = PetscViewerASCIISetTab(v,vtab); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A;
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt *ia,*ja;
MatPartitioning part;
IS is,isn;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 4) SETERRQ(PETSC_COMM_WORLD,1,"Must run with 4 processors");
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscMalloc1(5,&ia);CHKERRQ(ierr);
ierr = PetscMalloc1(16,&ja);CHKERRQ(ierr);
if (!rank) {
ja[0] = 1; ja[1] = 4; ja[2] = 0; ja[3] = 2; ja[4] = 5; ja[5] = 1; ja[6] = 3; ja[7] = 6;
ja[8] = 2; ja[9] = 7;
ia[0] = 0; ia[1] = 2; ia[2] = 5; ia[3] = 8; ia[4] = 10;
} else if (rank == 1) {
ja[0] = 0; ja[1] = 5; ja[2] = 8; ja[3] = 1; ja[4] = 4; ja[5] = 6; ja[6] = 9; ja[7] = 2;
ja[8] = 5; ja[9] = 7; ja[10] = 10; ja[11] = 3; ja[12] = 6; ja[13] = 11;
ia[0] = 0; ia[1] = 3; ia[2] = 7; ia[3] = 11; ia[4] = 14;
} else if (rank == 2) {
ja[0] = 4; ja[1] = 9; ja[2] = 12; ja[3] = 5; ja[4] = 8; ja[5] = 10; ja[6] = 13; ja[7] = 6;
ja[8] = 9; ja[9] = 11; ja[10] = 14; ja[11] = 7; ja[12] = 10; ja[13] = 15;
ia[0] = 0; ia[1] = 3; ia[2] = 7; ia[3] = 11; ia[4] = 14;
} else {
ja[0] = 8; ja[1] = 13; ja[2] = 9; ja[3] = 12; ja[4] = 14; ja[5] = 10; ja[6] = 13; ja[7] = 15;
ja[8] = 11; ja[9] = 14;
ia[0] = 0; ia[1] = 2; ia[2] = 5; ia[3] = 8; ia[4] = 10;
}
ierr = MatCreateMPIAdj(PETSC_COMM_WORLD,4,16,ia,ja,NULL,&A);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/*
Partition the graph of the matrix
*/
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* get new processor owner number of each vertex */
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
/* get new global number of each old global number */
ierr = ISPartitioningToNumbering(is,&isn);CHKERRQ(ierr);
ierr = ISView(isn,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = ISDestroy(&isn);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
MatType mtype = MATMPIAIJ; /* matrix format */
Mat A,B; /* matrix */
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
PetscBool flg,viewMats,viewIS,viewVecs;
PetscInt ierr,*nlocal,m,n;
PetscMPIInt rank,size;
MatPartitioning part;
IS is,isn;
Vec xin, xout;
VecScatter scat;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_mats", &viewMats);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_is", &viewIS);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_vecs", &viewVecs);CHKERRQ(ierr);
/*
Determine file from which we read the matrix
*/
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
/*
Open binary file. Note that we use FILE_MODE_READ to indicate
reading from this file.
*/
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
/*
Load the matrix and vector; then destroy the viewer.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,mtype);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&xin);CHKERRQ(ierr);
ierr = VecLoad(xin,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Original matrix:\n");
ierr = MatView(A,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
if (viewVecs) {
if (!rank) printf("Original vector:\n");
ierr = VecView(xin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Partition the graph of the matrix */
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* get new processor owner number of each vertex */
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS1 - new processor ownership:\n");
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* get new global number of each old global number */
ierr = ISPartitioningToNumbering(is,&isn);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS2 - new global numbering:\n");
ierr = ISView(isn,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* get number of new vertices for each processor */
ierr = PetscMalloc(size*sizeof(PetscInt),&nlocal);CHKERRQ(ierr);
ierr = ISPartitioningCount(is,size,nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
/* get old global number of each new global number */
ierr = ISInvertPermutation(isn,nlocal[rank],&is);CHKERRQ(ierr);
ierr = PetscFree(nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&isn);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS3=inv(IS2) - old global number of each new global number:\n");
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* move the matrix rows to the new processes they have been assigned to by the permutation */
ierr = ISSort(is);CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,is,is,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
/* move the vector rows to the new processes they have been assigned to */
ierr = MatGetLocalSize(B,&m,&n);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,m,PETSC_DECIDE,&xout);CHKERRQ(ierr);
ierr = VecScatterCreate(xin,is,xout,NULL,&scat);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,xin,xout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,xin,xout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Partitioned matrix:\n");
ierr = MatView(B,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
if (viewVecs) {
if (!rank) printf("Mapped vector:\n");
ierr = VecView(xout,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
{
PetscInt rstart,i,*nzd,*nzo,nzl,nzmax = 0,*ncols,nrow,j;
Mat J;
const PetscInt *cols;
const PetscScalar *vals;
PetscScalar *nvals;
ierr = MatGetOwnershipRange(B,&rstart,NULL);CHKERRQ(ierr);
ierr = PetscMalloc(2*m*sizeof(PetscInt),&nzd);CHKERRQ(ierr);
ierr = PetscMemzero(nzd,2*m*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMalloc(2*m*sizeof(PetscInt),&nzo);CHKERRQ(ierr);
ierr = PetscMemzero(nzo,2*m*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = MatGetRow(B,i+rstart,&nzl,&cols,NULL);CHKERRQ(ierr);
for (j=0; j<nzl; j++) {
if (cols[j] >= rstart && cols[j] < rstart+n) {
nzd[2*i] += 2;
nzd[2*i+1] += 2;
} else {
nzo[2*i] += 2;
nzo[2*i+1] += 2;
}
}
nzmax = PetscMax(nzmax,nzd[2*i]+nzo[2*i]);
ierr = MatRestoreRow(B,i+rstart,&nzl,&cols,NULL);CHKERRQ(ierr);
}
ierr = MatCreateAIJ(PETSC_COMM_WORLD,2*m,2*m,PETSC_DECIDE,PETSC_DECIDE,0,nzd,0,nzo,&J);CHKERRQ(ierr);
ierr = PetscInfo(0,"Created empty Jacobian matrix\n");CHKERRQ(ierr);
ierr = PetscFree(nzd);CHKERRQ(ierr);
ierr = PetscFree(nzo);CHKERRQ(ierr);
ierr = PetscMalloc2(nzmax,PetscInt,&ncols,nzmax,PetscScalar,&nvals);CHKERRQ(ierr);
ierr = PetscMemzero(nvals,nzmax*sizeof(PetscScalar));CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = MatGetRow(B,i+rstart,&nzl,&cols,&vals);CHKERRQ(ierr);
for (j=0; j<nzl; j++) {
ncols[2*j] = 2*cols[j];
ncols[2*j+1] = 2*cols[j]+1;
}
nrow = 2*(i+rstart);
ierr = MatSetValues(J,1,&nrow,2*nzl,ncols,nvals,INSERT_VALUES);CHKERRQ(ierr);
nrow = 2*(i+rstart) + 1;
ierr = MatSetValues(J,1,&nrow,2*nzl,ncols,nvals,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(B,i+rstart,&nzl,&cols,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Jacobian matrix structure:\n");
ierr = MatView(J,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = PetscFree2(ncols,nvals);CHKERRQ(ierr);
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecDestroy(&xin);CHKERRQ(ierr);
ierr = VecDestroy(&xout);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat C,A;
PetscInt i,j,m = 5,n = 5,Ii,J;
PetscErrorCode ierr;
PetscScalar v,five = 5.0,one = 1.0;
IS isrow,row,col;
Vec x,u,b;
PetscReal norm;
MatFactorInfo info;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,m*n,m*n,5,NULL,&C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
/* create the matrix for the five point stencil, YET AGAIN*/
for (i=0; i<m; i++) {
for (j=0; j<n; j++) {
v = -1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,(m*n)/2,0,2,&isrow);CHKERRQ(ierr);
ierr = MatZeroRowsIS(C,isrow,five,0,0);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,m*n,&u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&x);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b);CHKERRQ(ierr);
ierr = VecSet(u,one);CHKERRQ(ierr);
ierr = MatMultTranspose(C,u,b);CHKERRQ(ierr);
/* Set default ordering to be Quotient Minimum Degree; also read
orderings from the options database */
ierr = MatGetOrdering(C,MATORDERINGQMD,&row,&col);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);CHKERRQ(ierr);
ierr = MatGetFactor(C,MATSOLVERPETSC,MAT_FACTOR_LU,&A);CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(A,C,row,col,&info);CHKERRQ(ierr);
ierr = MatLUFactorNumeric(A,C,&info);CHKERRQ(ierr);
ierr = MatSolveTranspose(A,b,x);CHKERRQ(ierr);
ierr = ISView(row,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecAXPY(x,-1.0,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Norm of error %g\n",(double)norm);CHKERRQ(ierr);
ierr = ISDestroy(&row);CHKERRQ(ierr);
ierr = ISDestroy(&col);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da,*subda;
PetscInt i,dim=3;
PetscMPIInt size,rank;
Vec v;
Vec slvec,sgvec;
IS *ois,*iis;
VecScatter oscata;
VecScatter *iscat,*oscat,*gscat;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr);
ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr);
{
DMDALocalInfo subinfo;
MatStencil lower,upper;
IS patchis,subpatchis;
Vec smallvec;
Vec largevec;
VecScatter patchscat;
ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr);
lower.i = info.xs;
lower.j = info.ys;
lower.k = info.zs;
upper.i = info.xs+info.xm;
upper.j = info.ys+info.ym;
upper.k = info.zs+info.zm;
/* test the patch IS as a thing to scatter to/from */
ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr);
ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr);
ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr);
ierr = VecSet(largevec,0);CHKERRQ(ierr);
ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&smallvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = ISDestroy(&patchis);CHKERRQ(ierr);
ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr);
}
/* view the various parts */
{
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr);
/* test filling outer between the big DM and the small ones with the IS scatter*/
ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr);
ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* test the local-to-local scatter */
/* fill up the local subdomain and then add them together */
ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test ghost scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test overlap scattering backwards */
ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test interior scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test matrix allocation */
for (i = 0; i < size; i++) {
if (i == rank) {
Mat m;
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr);
ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&m);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr);
}
ierr = DMDestroy(&subda[0]);CHKERRQ(ierr);
ierr = ISDestroy(&ois[0]);CHKERRQ(ierr);
ierr = ISDestroy(&iis[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr);
ierr = PetscFree(iscat);CHKERRQ(ierr);
ierr = PetscFree(oscat);CHKERRQ(ierr);
ierr = PetscFree(gscat);CHKERRQ(ierr);
ierr = PetscFree(oscata);CHKERRQ(ierr);
ierr = PetscFree(subda);CHKERRQ(ierr);
ierr = PetscFree(ois);CHKERRQ(ierr);
ierr = PetscFree(iis);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
MPI_Comm comm;
MatType mtype;
PetscSF sf, sfDof, sfAdj;
PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj;
PetscInt nroots, nleaves, l, p;
const PetscInt *leaves;
const PetscSFNode *remotes;
PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols;
PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets;
PetscInt adjSize;
PetscLayout rLayout;
PetscInt locRows, rStart, rEnd, r;
PetscMPIInt size;
PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock;
PetscBool useAnchors;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3);
PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4);
PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
if (dnz) PetscValidPointer(dnz,5);
if (onz) PetscValidPointer(onz,6);
if (dnzu) PetscValidPointer(dnzu,7);
if (onzu) PetscValidPointer(onzu,8);
ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE;
ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr);
/* Create dof SF based on point SF */
if (debug) {
ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sf, NULL);CHKERRQ(ierr);
}
ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr);
}
/* Create section for dof adjacency (dof ==> # adj dof) */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr);
/* Fill in the ghost dofs on the interface */
ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr);
/* use constraints in finding adjacency in this routine */
ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr);
ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr);
/*
section - maps points to (# dofs, local dofs)
sectionGlobal - maps points to (# dofs, global dofs)
leafSectionAdj - maps unowned local dofs to # adj dofs
rootSectionAdj - maps owned local dofs to # adj dofs
adj - adj global dofs indexed by leafSectionAdj
rootAdj - adj global dofs indexed by rootSectionAdj
sf - describes shared points across procs
sfDof - describes shared dofs across procs
sfAdj - describes shared adjacent dofs across procs
** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point.
(0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj
(This is done in DMPlexComputeAnchorAdjacencies())
1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj
Reduce those counts to rootSectionAdj (now redundantly counting some interface points)
2. Visit owned points on interface, count adjacencies placing in rootSectionAdj
Create sfAdj connecting rootSectionAdj and leafSectionAdj
3. Visit unowned points on interface, write adjacencies to adj
Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies)
4. Visit owned points on interface, write adjacencies to rootAdj
Remove redundancy in rootAdj
** The last two traversals use transitive closure
5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj)
Allocate memory addressed by sectionAdj (cols)
6. Visit all owned points in the subdomain, insert dof adjacencies into cols
** Knowing all the column adjacencies, check ownership and sum into dnz and onz
*/
ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr);
ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get maximum remote adjacency sizes for owned dofs on interface (roots) */
if (doComm) {
ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
}
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Add in local adjacency sizes for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create adj SF based on dof SF */
ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr);
/* Create leaf adjacency */
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof, anOff;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt aoff, i = 0;
ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd;
++i;
}
}
for (q = 0; q < anDof; q++) {
adj[aoff+i] = anchorAdj[anOff+q];
++i;
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Gather adjacenct indices to root */
ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr);
for (r = 0; r < adjSize; ++r) rootAdj[r] = -1;
if (doComm) {
ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr);
ierr = PetscFree(adj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Add in local adjacency indices for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt adof, aoff, i;
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
i = adof-1;
for (q = 0; q < anDof; q++) {
rootAdj[aoff+i] = anchorAdj[anOff+q];
--i;
}
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
--i;
}
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Compress indices */
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof, off, d;
PetscInt adof, aoff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
for (d = off; d < off+dof-cdof; ++d) {
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr);
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Build adjacency section: Maps global indices to sets of adjacent global indices */
ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, noff;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get adjacent indices */
ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr);
ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
PetscInt aoff, roff;
ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr);
ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
PetscInt adof, aoff, i = 0;
ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
const PetscInt *ncind;
/* Adjacent points may not be in the section chart */
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd, ++i) {
cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
}
}
for (q = 0; q < anDof; q++, i++) {
cols[aoff+i] = anchorAdj[anOff + q];
}
if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p);
}
}
ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr);
ierr = PetscFree(anchorAdj);CHKERRQ(ierr);
ierr = PetscFree(rootAdj);CHKERRQ(ierr);
ierr = PetscFree(tmpAdj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Create allocation vectors from adjacency graph */
ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr);
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
/* Only loop over blocks of rows */
if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs);
for (r = rStart/bs; r < rEnd/bs; ++r) {
const PetscInt row = r*bs;
PetscInt numCols, cStart, c;
ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr);
for (c = cStart; c < cStart+numCols; ++c) {
if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) {
++dnz[r-rStart];
if (cols[c] >= row) ++dnzu[r-rStart];
} else {
++onz[r-rStart];
if (cols[c] >= row) ++onzu[r-rStart];
}
}
}
if (bs > 1) {
for (r = 0; r < locRows/bs; ++r) {
dnz[r] /= bs;
onz[r] /= bs;
dnzu[r] /= bs;
onzu[r] /= bs;
}
}
/* Set matrix pattern */
ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr);
ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* Check for symmetric storage */
ierr = MatGetType(A, &mtype);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);}
/* Fill matrix with zeros */
if (fillMatrix) {
PetscScalar *values;
PetscInt maxRowLen = 0;
for (r = rStart; r < rEnd; ++r) {
PetscInt len;
ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr);
maxRowLen = PetscMax(maxRowLen, len);
}
ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr);
for (r = rStart; r < rEnd; ++r) {
PetscInt numCols, cStart;
ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr);
ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* restore original useAnchors */
ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat C;
PetscInt i,j,m = 3,n = 3,Ii,J;
PetscErrorCode ierr;
PetscBool flg;
PetscScalar v;
IS perm,iperm;
Vec x,u,b,y;
PetscReal norm,tol=PETSC_SMALL;
MatFactorInfo info;
PetscMPIInt size;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,1,"This is a uniprocessor example only!");
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-symmetric",&flg);CHKERRQ(ierr);
if (flg) { /* Treat matrix as symmetric only if we set this flag */
ierr = MatSetOption(C,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatSetOption(C,MAT_SYMMETRY_ETERNAL,PETSC_TRUE);CHKERRQ(ierr);
}
/* Create the matrix for the five point stencil, YET AGAIN */
for (i=0; i<m; i++) {
for (j=0; j<n; j++) {
v = -1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatGetOrdering(C,MATORDERINGRCM,&perm,&iperm);CHKERRQ(ierr);
ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(perm,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,m*n,&u);CHKERRQ(ierr);
ierr = VecSet(u,1.0);CHKERRQ(ierr);
ierr = VecDuplicate(u,&x);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b);CHKERRQ(ierr);
ierr = VecDuplicate(u,&y);CHKERRQ(ierr);
ierr = MatMult(C,u,b);CHKERRQ(ierr);
ierr = VecCopy(b,y);CHKERRQ(ierr);
ierr = VecScale(y,2.0);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_FROBENIUS,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Frobenius norm of matrix %g\n",(double)norm);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"One norm of matrix %g\n",(double)norm);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_INFINITY,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Infinity norm of matrix %g\n",(double)norm);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);CHKERRQ(ierr);
info.fill = 2.0;
info.dtcol = 0.0;
info.zeropivot = 1.e-14;
info.pivotinblocks = 1.0;
ierr = MatLUFactor(C,perm,iperm,&info);CHKERRQ(ierr);
/* Test MatSolve */
ierr = MatSolve(C,b,x);CHKERRQ(ierr);
ierr = VecView(b,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecAXPY(x,-1.0,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatSolve: Norm of error %g\n",(double)norm);CHKERRQ(ierr);
}
/* Test MatSolveAdd */
ierr = MatSolveAdd(C,b,y,x);CHKERRQ(ierr);
ierr = VecAXPY(x,-1.0,y);CHKERRQ(ierr);
ierr = VecAXPY(x,-1.0,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatSolveAdd(): Norm of error %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = ISDestroy(&iperm);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
const struct {PetscInt i,j; PetscScalar v;} entries[] = {{0,3,1.},{1,2,2.},{2,1,3.},{2,4,4.},{3,0,5.},{3,3,6.},{4,1,7.},{4,4,8.}};
const PetscInt ixrow[5] = {4,2,1,3,0},ixcol[5] = {3,2,1,4,0};
Mat A,B;
PetscErrorCode ierr;
PetscInt i,rstart,rend,cstart,cend;
IS isrow,iscol;
PetscViewer viewer,sviewer;
PetscBool view_sparse;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* ------- Assemble matrix, --------- */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,5,5);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(A,&cstart,&cend);CHKERRQ(ierr);
for (i=0; i<(PetscInt)(sizeof(entries)/sizeof(entries[0])); i++) {
ierr = MatSetValue(A,entries[i].i,entries[i].j,entries[i].v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* ------ Prepare index sets ------ */
ierr = ISCreateGeneral(PETSC_COMM_WORLD,rend-rstart,ixrow+rstart,PETSC_USE_POINTER,&isrow);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,5,ixcol,PETSC_USE_POINTER,&iscol);CHKERRQ(ierr);
ierr = ISSetPermutation(isrow);CHKERRQ(ierr);
ierr = ISSetPermutation(iscol);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
view_sparse = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL, "-view_sparse", &view_sparse, PETSC_NULL);CHKERRQ(ierr);
if (!view_sparse) {
ierr = PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_DENSE);CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix\n");CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = MatPermute(A,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Permuted matrix\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Column permutation\n");CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(viewer,&sviewer);CHKERRQ(ierr);
ierr = ISView(iscol,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(viewer,&sviewer);CHKERRQ(ierr);
/* Free data structures */
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char *argv[])
{
PetscErrorCode ierr;
Mat A,S = NULL,Sexplicit = NULL;
IS is0,is1;
ierr = PetscInitialize(&argc,&argv,0,help);
CHKERRQ(ierr);
/* Test the Schur complement one way */
ierr = Create(PETSC_COMM_WORLD,&A,&is0,&is1);
CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = ISView(is0,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = ISView(is1,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = MatGetSchurComplement(A,is0,is0,is1,is1,MAT_INITIAL_MATRIX,&S,MAT_SCHUR_COMPLEMENT_AINV_LUMP,MAT_IGNORE_MATRIX,NULL);
CHKERRQ(ierr);
ierr = MatComputeExplicitOperator(S,&Sexplicit);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nExplicit Schur complement of (0,0) in (1,1)\n");
CHKERRQ(ierr);
ierr = MatView(Sexplicit,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = Destroy(&A,&is0,&is1);
CHKERRQ(ierr);
ierr = MatDestroy(&S);
CHKERRQ(ierr);
ierr = MatDestroy(&Sexplicit);
CHKERRQ(ierr);
/* And the other */
ierr = Create(PETSC_COMM_WORLD,&A,&is0,&is1);
CHKERRQ(ierr);
ierr = MatGetSchurComplement(A,is1,is1,is0,is0,MAT_INITIAL_MATRIX,&S,MAT_SCHUR_COMPLEMENT_AINV_LUMP,MAT_IGNORE_MATRIX,NULL);
CHKERRQ(ierr);
ierr = MatComputeExplicitOperator(S,&Sexplicit);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nExplicit Schur complement of (1,1) in (0,0)\n");
CHKERRQ(ierr);
ierr = MatView(Sexplicit,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = Destroy(&A,&is0,&is1);
CHKERRQ(ierr);
ierr = MatDestroy(&S);
CHKERRQ(ierr);
ierr = MatDestroy(&Sexplicit);
CHKERRQ(ierr);
/* This time just the preconditioner */
ierr = Create(PETSC_COMM_WORLD,&A,&is0,&is1);
CHKERRQ(ierr);
ierr = MatGetSchurComplement(A,is0,is0,is1,is1,MAT_IGNORE_MATRIX,NULL,MAT_SCHUR_COMPLEMENT_AINV_LUMP,MAT_INITIAL_MATRIX,&S);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPreconditioning Schur complement of (0,0) in (1,1)\n");
CHKERRQ(ierr);
ierr = MatView(S,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
/* Modify and refresh */
ierr = MatShift(A,1.);
CHKERRQ(ierr);
ierr = MatGetSchurComplement(A,is0,is0,is1,is1,MAT_IGNORE_MATRIX,NULL,MAT_SCHUR_COMPLEMENT_AINV_LUMP,MAT_REUSE_MATRIX,&S);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nAfter update\n");
CHKERRQ(ierr);
ierr = MatView(S,PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = Destroy(&A,&is0,&is1);
CHKERRQ(ierr);
ierr = MatDestroy(&S);
CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/* Distribute cones
- Partitioning: input partition point map and naive sf, output sf with inverse of map, distribute points
- Distribute section: input current sf, communicate sizes and offsets, output local section and offsets (only use for new sf)
- Create SF for values: input current sf and offsets, output new sf
- Distribute values: input new sf, communicate values
*/
PetscErrorCode DistributeMesh(DM dm, AppCtx *user, PetscSF *pointSF, DM *parallelDM)
{
MPI_Comm comm = ((PetscObject) dm)->comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS cellPart, part;
PetscSection cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF;
ISLocalToGlobalMapping renumbering;
PetscSF coneSF;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets, newConesSize;
PetscInt *cones, *newCones;
PetscMPIInt numProcs, rank, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMMeshGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = DMMeshCreatePartition(dm, &cellPartSection, &cellPart, height);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) {
numRemoteRanks = numProcs;
} else {
numRemoteRanks = 0;
}
ierr = PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);CHKERRQ(ierr);
for(p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, PETSC_NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(partSF, PETSC_NULL);CHKERRQ(ierr);
/* Close the partition over the mesh */
ierr = DMMeshCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMMeshCreate(comm, parallelDM);CHKERRQ(ierr);
ierr = DMMeshSetDimension(*parallelDM, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *parallelDM, "Parallel Mesh");CHKERRQ(ierr);
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, pointSF);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(*pointSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, PETSC_NULL);CHKERRQ(ierr);
/* Cleanup */
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMMeshGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMMeshGetConeSection(*parallelDM, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(*pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMMeshSetUp(*parallelDM);CHKERRQ(ierr);
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(*pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMMeshGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMMeshGetCones(*parallelDM, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, PETSC_NULL, newCones);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
/* Create supports and stratify sieve */
ierr = DMMeshSymmetrize(*parallelDM);CHKERRQ(ierr);
ierr = DMMeshStratify(*parallelDM);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
uses block index sets\n\n";
/* 'mpiexec -n 3 ./ex2 -vecscatter_type mpi3node' might give incorrect solution due to multiple cores write to the same variable */
#include <petscvec.h>
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt bs=1,n=5,i,low;
PetscInt ix0[3] = {5,7,9},iy0[3] = {1,2,4},ix1[3] = {2,3,4},iy1[3] = {0,1,3};
PetscMPIInt size,rank;
PetscScalar *array;
Vec x,y;
IS isx,isy;
VecScatter ctx;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (size <2) SETERRQ(PETSC_COMM_SELF,1,"Must run more than one processor");
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
n = bs*n;
/* Create vector x over shared memory */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(x,VECNODE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&low,NULL);CHKERRQ(ierr);
ierr = VecGetArray(x,&array);CHKERRQ(ierr);
for (i=0; i<n; i++) {
array[i] = (PetscScalar)(i + low);
}
ierr = VecRestoreArray(x,&array);CHKERRQ(ierr);
/* Create a sequential vector y */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&y);CHKERRQ(ierr);
ierr = VecSet(y,0.0);CHKERRQ(ierr);
/* Create two index sets */
if (!rank) {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix0,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy0,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
} else {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix1,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy1,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
}
if (rank == 10) {
ierr = PetscPrintf(PETSC_COMM_SELF,"\n[%d] isx:\n",rank);CHKERRQ(ierr);
ierr = ISView(isx,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = VecScatterCreateWithData(x,isx,y,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterSetFromOptions(ctx);CHKERRQ(ierr);
/* Test forward vecscatter */
ierr = VecScatterBegin(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (rank == 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] y:\n",rank);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
/* Test reverse vecscatter */
ierr = VecScale(y,-1.0);CHKERRQ(ierr);
if (rank) {
ierr = VecScale(y,1.0/(size - 1));CHKERRQ(ierr);
}
ierr = VecScatterBegin(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Free spaces */
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
PetscBool flg,TestOverlap,TestSubMat,TestAllcols,test_sorted=PETSC_FALSE;
PetscInt vid = -1;
#if defined(PETSC_USE_LOG)
PetscLogStage stages[2];
#endif
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_sorted",&test_sorted,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<Mbs; i++) {
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatEqual(A, Atrans, &flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
if (vid >= 0 && vid < size) {
if (!rank) printf("A: \n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("sA: \n");
ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test sA==A through MatMult() */
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA");
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
if (!TestAllcols) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
if (rank == vid) {
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
} else { /* Test all rows and colums */
sz = M;
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr);
if (rank == vid) {
PetscBool colflag;
ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr);
printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
}
}
ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);CHKERRQ(ierr);
/* Test MatIncreaseOverlap */
if (TestOverlap) {
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (rank == vid) {
printf("\n[%d] IS from BAIJ:\n",rank);
ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
printf("\n[%d] IS from SBAIJ:\n",rank);
ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
if (!rank) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
ierr = ISSort(is2[i]);CHKERRQ(ierr);
/* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
}
SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i);
}
}
}
/* Test MatCreateSubmatrices */
if (TestSubMat) {
if (test_sorted) {
for (i = 0; i < nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
}
}
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA");
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatCreateSubmatrices(): A != sA");
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatsA);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
