int main(int argc,char **args)
{
Mat A,B;
PetscInt i,rstart,rend;
PetscMPIInt rank,size;
PetscErrorCode ierr;
PetscScalar v;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* Create a MPIBAIJ matrix */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,32,32);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPIBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,2,2,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,2,2,NULL,2,NULL);CHKERRQ(ierr);
v = 1.0;
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatSetValues(A,1,&i,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Convert A to AIJ format */
ierr = MatConvert(A,MATAIJ,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode Assemble(MPI_Comm comm,PetscInt bs,const MatType mtype)
{
const PetscInt rc[] = {0,1,2,3};
const PetscScalar vals[] = {1, 2, 3, 4, 5, 6, 7, 8,
9,10,11,12,13,14,15,16,
17,18,19,20,21,22,23,24,
25,26,27,28,29,30,31,32,
33,34,35,36,37,38,39,40,
41,42,43,44,45,46,47,48,
49,50,51,52,53,54,55,56,
57,58,49,60,61,62,63,64};
Mat A;
PetscViewer viewer;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MatCreate(comm,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,4*bs,4*bs);CHKERRQ(ierr);
ierr = MatSetType(A,mtype);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,2,NULL,2,NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,2,NULL,2,NULL);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* All processes contribute a global matrix */
ierr = MatSetValuesBlocked(A,4,rc,4,rc,vals,ADD_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Matrix %s(%D)\n",mtype,bs);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
Mat A;
MatPartitioning part;
IS is;
PetscInt i,m,N,rstart,rend,nemptyranks,*emptyranks,nbigranks,*bigranks;
PetscMPIInt rank,size;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
nemptyranks = 10;
nbigranks = 10;
ierr = PetscMalloc2(nemptyranks,PetscInt,&emptyranks,nbigranks,PetscInt,&bigranks);CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"Partitioning example options",PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-emptyranks","Ranks to be skipped by partition","",emptyranks,&nemptyranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-bigranks","Ranks to be overloaded","",bigranks,&nbigranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
m = 1;
for (i=0; i<nemptyranks; i++) if (rank == emptyranks[i]) m = 0;
for (i=0; i<nbigranks; i++) if (rank == bigranks[i]) m = 5;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m,m,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,1,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,1,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,1,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetSize(A,PETSC_NULL,&N);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
const PetscInt cols[] = {(i+N-1)%N,i,(i+1)%N};
const PetscScalar vals[] = {1,1,1};
ierr = MatSetValues(A,1,&i,3,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree2(emptyranks,bigranks);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
MatXAIJSetPreallocation - set preallocation for serial and parallel AIJ, BAIJ, and SBAIJ matrices
Collective on Mat
Input Arguments:
+ A - matrix being preallocated
. bs - block size
. dnnz - number of nonzero blocks per block row of diagonal part of parallel matrix
. onnz - number of nonzero blocks per block row of off-diagonal part of parallel matrix
. dnnzu - number of nonzero blocks per block row of upper-triangular part of diagonal part of parallel matrix
- onnzu - number of nonzero blocks per block row of upper-triangular part of off-diagonal part of parallel matrix
Level: beginner
.seealso: MatSeqAIJSetPreallocation(), MatMPIAIJSetPreallocation(), MatSeqBAIJSetPreallocation(), MatMPIBAIJSetPreallocation(), MatSeqSBAIJSetPreallocation(), MatMPISBAIJSetPreallocation(),
PetscSplitOwnership()
@*/
PetscErrorCode MatXAIJSetPreallocation(Mat A,PetscInt bs,const PetscInt dnnz[],const PetscInt onnz[],const PetscInt dnnzu[],const PetscInt onnzu[])
{
PetscErrorCode ierr;
void (*aij)(void);
PetscFunctionBegin;
ierr = MatSetBlockSize(A,bs);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->rmap);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);
CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,0,dnnz,0,onnz);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,bs,0,dnnzu);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,0,dnnzu,0,onnzu);
CHKERRQ(ierr);
/*
In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
good before going on with it.
*/
ierr = PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(A,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz,0,onnz);
CHKERRQ(ierr);
} else { /* Convert block-row precallocation to scalar-row */
PetscInt i,m,*sdnnz,*sonnz;
ierr = MatGetLocalSize(A,&m,NULL);
CHKERRQ(ierr);
ierr = PetscMalloc2((!!dnnz)*m,PetscInt,&sdnnz,(!!onnz)*m,PetscInt,&sonnz);
CHKERRQ(ierr);
for (i=0; i<m; i++) {
if (dnnz) sdnnz[i] = dnnz[i/bs] * bs;
if (onnz) sonnz[i] = onnz[i/bs] * bs;
}
ierr = MatSeqAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL,0,onnz ? sonnz : NULL);
CHKERRQ(ierr);
ierr = PetscFree2(sdnnz,sonnz);
CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
void linearSystemPETSc<scalar>::preAllocateEntries()
{
if (_entriesPreAllocated) return;
if (!_isAllocated) Msg::Fatal("system must be allocated first");
int blockSize = _getBlockSizeFromParameters();
std::vector<int> nByRowDiag (_localSize), nByRowOffDiag (_localSize);
if (_sparsity.getNbRows() == 0) {
PetscInt prealloc = 500;
PetscBool set;
PetscOptionsGetInt(PETSC_NULL, "-petsc_prealloc", &prealloc, &set);
prealloc = std::min(prealloc, _localSize);
nByRowDiag.resize(0);
nByRowDiag.resize(_localSize, prealloc);
} else {
for (int i = 0; i < _localSize; i++) {
int n;
const int *r = _sparsity.getRow(i, n);
for (int j = 0; j < n; j++) {
if (r[j] >= _localRowStart && r[j] < _localRowEnd)
nByRowDiag[i] ++;
else
nByRowOffDiag[i] ++;
}
}
_sparsity.clear();
}
//MatXAIJSetPreallocation is not available in petsc < 3.3
int commSize = 1;
MPI_Comm_size(_comm, &commSize);
if (commSize == 1){
if (blockSize == 1)
_try(MatSeqAIJSetPreallocation(_a, 0, &nByRowDiag[0]));
else
_try(MatSeqBAIJSetPreallocation(_a, blockSize, 0, &nByRowDiag[0]));
}
else {
if (blockSize == 1)
_try(MatMPIAIJSetPreallocation(_a, 0, &nByRowDiag[0], 0, &nByRowOffDiag[0]));
else
_try(MatMPIBAIJSetPreallocation(_a, blockSize, 0, &nByRowDiag[0], 0, &nByRowOffDiag[0]));
}
if (blockSize > 1)
_try(MatSetOption(_a, MAT_ROW_ORIENTED, PETSC_FALSE));
_entriesPreAllocated = true;
}
static PetscErrorCode DMCreateMatrix_Redundant(DM dm,Mat *J)
{
DM_Redundant *red = (DM_Redundant*)dm->data;
PetscErrorCode ierr;
ISLocalToGlobalMapping ltog,ltogb;
PetscInt i,rstart,rend,*cols;
PetscScalar *vals;
PetscFunctionBegin;
ierr = MatCreate(PetscObjectComm((PetscObject)dm),J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,red->n,red->n,red->N,red->N);CHKERRQ(ierr);
ierr = MatSetType(*J,dm->mattype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*J,red->n,NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,1,red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,1,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(dm,<og);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMappingBlock(dm,<ogb);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,ltogb,ltogb);CHKERRQ(ierr);
ierr = PetscMalloc2(red->N,&cols,red->N,&vals);CHKERRQ(ierr);
for (i=0; i<red->N; i++) {
cols[i] = i;
vals[i] = 0.0;
}
ierr = MatGetOwnershipRange(*J,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatSetValues(*J,1,&i,red->N,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree2(cols,vals);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatISSetMPIXAIJPreallocation_Private(Mat A, Mat B, PetscBool maxreduce)
{
Mat_IS *matis = (Mat_IS*)(A->data);
PetscInt *my_dnz,*my_onz,*dnz,*onz,*mat_ranges,*row_ownership;
const PetscInt *global_indices_r,*global_indices_c;
PetscInt i,j,bs,rows,cols;
PetscInt lrows,lcols;
PetscInt local_rows,local_cols;
PetscMPIInt nsubdomains;
PetscBool isdense,issbaij;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&nsubdomains);CHKERRQ(ierr);
ierr = MatGetSize(A,&rows,&cols);CHKERRQ(ierr);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (A->rmap->mapping != A->cmap->mapping) {
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
} else {
global_indices_c = global_indices_r;
}
if (issbaij) {
ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
/*
An SF reduce is needed to sum up properly on shared rows.
Note that generally preallocation is not exact, since it overestimates nonzeros
*/
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(A);CHKERRQ(ierr);
}
ierr = MatGetLocalSize(A,&lrows,&lcols);CHKERRQ(ierr);
ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)A),lrows,lcols,dnz,onz);CHKERRQ(ierr);
/* All processes need to compute entire row ownership */
ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(A,(const PetscInt**)&mat_ranges);CHKERRQ(ierr);
for (i=0;i<nsubdomains;i++) {
for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
row_ownership[j] = i;
}
}
/*
my_dnz and my_onz contains exact contribution to preallocation from each local mat
then, they will be summed up properly. This way, preallocation is always sufficient
*/
ierr = PetscCalloc2(local_rows,&my_dnz,local_rows,&my_onz);CHKERRQ(ierr);
/* preallocation as a MATAIJ */
if (isdense) { /* special case for dense local matrices */
for (i=0;i<local_rows;i++) {
PetscInt index_row = global_indices_r[i];
for (j=i;j<local_rows;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[j];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (i != j) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1;
} else {
my_onz[j] += 1;
}
}
}
}
} else { /* TODO: this could be optimized using MatGetRowIJ */
for (i=0;i<local_rows;i++) {
const PetscInt *cols;
PetscInt ncols,index_row = global_indices_r[i];
ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[cols[j]];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (issbaij && index_col != index_row) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[cols[j]] += 1;
} else {
my_onz[cols[j]] += 1;
}
}
}
ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (global_indices_c != global_indices_r) {
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
}
ierr = PetscFree(row_ownership);CHKERRQ(ierr);
/* Reduce my_dnz and my_onz */
if (maxreduce) {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
} else {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
}
ierr = PetscFree2(my_dnz,my_onz);CHKERRQ(ierr);
/* Resize preallocation if overestimated */
for (i=0;i<lrows;i++) {
dnz[i] = PetscMin(dnz[i],lcols);
onz[i] = PetscMin(onz[i],cols-lcols);
}
/* set preallocation */
ierr = MatMPIAIJSetPreallocation(B,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i*bs]/bs;
onz[i] = onz[i*bs]/bs;
}
ierr = MatMPIBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr);
if (issbaij) {
ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
PetscBool flg,TestOverlap,TestSubMat,TestAllcols,test_sorted=PETSC_FALSE;
PetscInt vid = -1;
#if defined(PETSC_USE_LOG)
PetscLogStage stages[2];
#endif
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_sorted",&test_sorted,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<Mbs; i++) {
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatEqual(A, Atrans, &flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
if (vid >= 0 && vid < size) {
if (!rank) printf("A: \n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("sA: \n");
ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test sA==A through MatMult() */
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA");
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
if (!TestAllcols) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
if (rank == vid) {
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
} else { /* Test all rows and colums */
sz = M;
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr);
if (rank == vid) {
PetscBool colflag;
ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr);
printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
}
}
ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);CHKERRQ(ierr);
/* Test MatIncreaseOverlap */
if (TestOverlap) {
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (rank == vid) {
printf("\n[%d] IS from BAIJ:\n",rank);
ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
printf("\n[%d] IS from SBAIJ:\n",rank);
ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
if (!rank) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
ierr = ISSort(is2[i]);CHKERRQ(ierr);
/* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
}
SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i);
}
}
}
/* Test MatCreateSubmatrices */
if (TestSubMat) {
if (test_sorted) {
for (i = 0; i < nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
}
}
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA");
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatCreateSubmatrices(): A != sA");
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatsA);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@C
SlicedGetMatrix - Creates a matrix with the correct parallel layout required for
computing the Jacobian on a function defined using the informatin in Sliced.
Collective on Sliced
Input Parameter:
+ slice - the slice object
- mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, MATSEQSBAIJ, MATMPISBAIJ,
or any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).
Output Parameters:
. J - matrix with the correct nonzero preallocation
(obviously without the correct Jacobian values)
Level: advanced
Notes: This properly preallocates the number of nonzeros in the sparse matrix so you
do not need to do it yourself.
.seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), DASetBlockFills()
@*/
PetscErrorCode PETSCDM_DLLEXPORT SlicedGetMatrix(Sliced slice, const MatType mtype,Mat *J)
{
PetscErrorCode ierr;
PetscInt *globals,*sd_nnz,*so_nnz,rstart,bs,i;
ISLocalToGlobalMapping lmap,blmap;
void (*aij)(void) = PETSC_NULL;
PetscFunctionBegin;
bs = slice->bs;
ierr = MatCreate(((PetscObject)slice)->comm,J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,slice->n*bs,slice->n*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(*J,mtype);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);CHKERRQ(ierr);
/* In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
* good before going on with it. */
ierr = PetscObjectQueryFunction((PetscObject)*J,"MatMPIAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)*J,"MatSeqAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);CHKERRQ(ierr);
} else if (!slice->d_nnz) {
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL,slice->o_nz*bs,PETSC_NULL);CHKERRQ(ierr);
} else {
/* The user has provided preallocation per block-row, convert it to per scalar-row respecting SlicedSetBlockFills() if applicable */
ierr = PetscMalloc2(slice->n*bs,PetscInt,&sd_nnz,(!!slice->o_nnz)*slice->n*bs,PetscInt,&so_nnz);CHKERRQ(ierr);
for (i=0; i<slice->n*bs; i++) {
sd_nnz[i] = (slice->d_nnz[i/bs]-1) * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs)
+ (slice->dfill ? slice->dfill->i[i%bs+1]-slice->dfill->i[i%bs] : bs);
if (so_nnz) {
so_nnz[i] = slice->o_nnz[i/bs] * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs);
}
}
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz,slice->o_nz*bs,so_nnz);CHKERRQ(ierr);
ierr = PetscFree2(sd_nnz,so_nnz);CHKERRQ(ierr);
}
}
ierr = MatSetBlockSize(*J,bs);CHKERRQ(ierr);
/* Set up the local to global map. For the scalar map, we have to translate to entry-wise indexing instead of block-wise. */
ierr = PetscMalloc((slice->n+slice->Nghosts)*bs*sizeof(PetscInt),&globals);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(*J,&rstart,PETSC_NULL);CHKERRQ(ierr);
for (i=0; i<slice->n*bs; i++) {
globals[i] = rstart + i;
}
for (i=0; i<slice->Nghosts*bs; i++) {
globals[slice->n*bs+i] = slice->ghosts[i/bs]*bs + i%bs;
}
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,(slice->n+slice->Nghosts)*bs,globals,&lmap);CHKERRQ(ierr);
ierr = PetscFree(globals);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingBlock(lmap,bs,&blmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,lmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,blmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(lmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(blmap);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETSC_STDCALL matmpibaijsetpreallocation_(Mat *mat,PetscInt *bs,PetscInt *d_nz,PetscInt *d_nnz,PetscInt *o_nz,PetscInt *o_nnz,PetscErrorCode *ierr)
{
CHKFORTRANNULLINTEGER(d_nnz);
CHKFORTRANNULLINTEGER(o_nnz);
*ierr = MatMPIBAIJSetPreallocation(*mat,*bs,*d_nz,d_nnz,*o_nz,o_nnz);
}
Solver::Petsc::Petsc (Grid& gr)
{
world = PETSC_COMM_WORLD;
MPI_Comm_rank (world, &rank);
MPI_Comm_size (world, &nProcs);
n = gr.n_in_elm;
bs = N_VAR;
vecGlobalSize = n*bs;
//DX = (double*) malloc (vecGlobalSize*sizeof(double));
//PetscMalloc1 (xGlobalSize, &DX);
DX = new double [vecGlobalSize];
// set x
VecCreate (world, &x);
VecSetType (x, VECSTANDARD);
VecSetBlockSize(x, bs);
VecSetSizes (x, PETSC_DECIDE, vecGlobalSize);
VecGetLocalSize (x, &vecLocalSize);
VecGetOwnershipRange (x, &vecLocBeg, &vecLocEnd);
// set b
VecDuplicate (x, &b);
// set A
MatCreate (world, &A);
MatSetType (A, MATMPIBAIJ);
//MatSetType (A, MATMPIAIJ);
MatSetSizes (A, PETSC_DECIDE, PETSC_DECIDE, vecGlobalSize, vecGlobalSize);
//MatSeqBAIJSetPreallocation (A, bs, 4, NULL);
//MatSeqAIJSetPreallocation (A, 4*bs, NULL);
//MatMPIAIJSetPreallocation (A, 4*bs, NULL, 4*bs, NULL);
MatMPIBAIJSetPreallocation (A, bs, 5, NULL, 5, NULL);
//MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
//MatCreateBAIJ (world, bs, PETSC_DECIDE, PETSC_DECIDE, n*bs, n*bs, 1, NULL, 3, NULL, &A);
// specific to pentagonal mesh . change later
MatGetOwnershipRange (A, &matLocBeg, &matLocEnd);
MatGetLocalSize (A, &matLocalSize, NULL);
KSPCreate (world, &ksp);
KSPSetOperators (ksp, A, A);
KSPGetPC (ksp, &pc);
PCSetType (pc, PCSOR);
KSPSetType (ksp, KSPGMRES);
KSPSetFromOptions (ksp);
localSizes = new int [nProcs];
int recvcounts[nProcs];
int displs[nProcs];
displs[0] = 0;
for (int i=0; i<nProcs; ++i)
{
recvcounts[i] = 1;
}
for (int i=1; i<nProcs; ++i)
{
displs[i] = displs[i-1] + recvcounts[i-1];
}
MPI_Allgatherv (&vecLocalSize, 1, MPI_INT, localSizes, recvcounts, displs, MPI_INT, world);
}
int main(int argc,char **args)
{
Mat A,B,*submatA,*submatB;
PetscInt bs=1,m=11,ov=1,i,j,k,*rows,*cols,nd=5,*idx,rstart,rend,sz,mm,nn,M,N,Mbs;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscScalar *vals,rval;
IS *is1,*is2;
PetscRandom rdm;
Vec xx,s1,s2;
PetscReal s1norm,s2norm,rnorm,tol = 100*PETSC_SMALL;
PetscBool flg,test_nd0=PETSC_FALSE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_size",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_nd0",&test_nd0,NULL);CHKERRQ(ierr);
/* Create a AIJ matrix A */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m*bs,m*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
/* Create a BAIJ matrix B */
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m*bs,m*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(B,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(B,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(B,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (i=0; i<40*bs; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[0] = bs*(int)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(int)(PetscRealPart(rval)*m);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(B,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
if (!rank && test_nd0) nd = 0; /* test case */
for (i=0; i<nd; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
sz = (int)(PetscRealPart(rval)*m);
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(int)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
}
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(B,nd,is2,ov);CHKERRQ(ierr);
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_SELF,"i=%D, flg=%d :bs=%D m=%D ov=%D nd=%D np=%D\n",i,flg,bs,m,ov,nd,size);CHKERRQ(ierr);
}
}
for (i=0; i<nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
ierr = ISSort(is2[i]);CHKERRQ(ierr);
}
ierr = MatCreateSubMatrices(B,nd,is2,is2,MAT_INITIAL_MATRIX,&submatB);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatB[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d]Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",rank,s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(B,nd,is2,is2,MAT_REUSE_MATRIX,&submatB);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatB[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d]Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",rank,s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatB);CHKERRQ(ierr);
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void PetscMatrix<T>::init ()
{
libmesh_assert(this->_dof_map);
// Clear initialized matrices
if (this->initialized())
this->clear();
this->_is_initialized = true;
const numeric_index_type my_m = this->_dof_map->n_dofs();
const numeric_index_type my_n = my_m;
const numeric_index_type n_l = this->_dof_map->n_dofs_on_processor(this->processor_id());
const numeric_index_type m_l = n_l;
const std::vector<numeric_index_type>& n_nz = this->_dof_map->get_n_nz();
const std::vector<numeric_index_type>& n_oz = this->_dof_map->get_n_oz();
// Make sure the sparsity pattern isn't empty unless the matrix is 0x0
libmesh_assert_equal_to (n_nz.size(), m_l);
libmesh_assert_equal_to (n_oz.size(), m_l);
PetscErrorCode ierr = 0;
PetscInt m_global = static_cast<PetscInt>(my_m);
PetscInt n_global = static_cast<PetscInt>(my_n);
PetscInt m_local = static_cast<PetscInt>(m_l);
PetscInt n_local = static_cast<PetscInt>(n_l);
ierr = MatCreate(this->comm().get(), &_mat);
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetSizes(_mat, m_local, n_local, m_global, n_global);
LIBMESH_CHKERRABORT(ierr);
#ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
PetscInt blocksize = static_cast<PetscInt>(this->_dof_map->block_size());
if (blocksize > 1)
{
// specified blocksize, bs>1.
// double check sizes.
libmesh_assert_equal_to (m_local % blocksize, 0);
libmesh_assert_equal_to (n_local % blocksize, 0);
libmesh_assert_equal_to (m_global % blocksize, 0);
libmesh_assert_equal_to (n_global % blocksize, 0);
ierr = MatSetType(_mat, MATBAIJ); // Automatically chooses seqbaij or mpibaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetBlockSize(_mat, blocksize);
LIBMESH_CHKERRABORT(ierr);
// transform the per-entry n_nz and n_oz arrays into their block counterparts.
std::vector<numeric_index_type> b_n_nz, b_n_oz;
transform_preallocation_arrays (blocksize,
n_nz, n_oz,
b_n_nz, b_n_oz);
ierr = MatSeqBAIJSetPreallocation(_mat, blocksize, 0, (PetscInt*)(b_n_nz.empty()?NULL:&b_n_nz[0]));
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIBAIJSetPreallocation(_mat, blocksize,
0, (PetscInt*)(b_n_nz.empty()?NULL:&b_n_nz[0]),
0, (PetscInt*)(b_n_oz.empty()?NULL:&b_n_oz[0]));
LIBMESH_CHKERRABORT(ierr);
}
else
#endif
{
// no block storage case
ierr = MatSetType(_mat, MATAIJ); // Automatically chooses seqaij or mpiaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSeqAIJSetPreallocation(_mat, 0, (PetscInt*)(n_nz.empty()?NULL:&n_nz[0]));
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIAIJSetPreallocation(_mat,
0, (PetscInt*)(n_nz.empty()?NULL:&n_nz[0]),
0, (PetscInt*)(n_oz.empty()?NULL:&n_oz[0]));
LIBMESH_CHKERRABORT(ierr);
}
// Is prefix information available somewhere? Perhaps pass in the system name?
ierr = MatSetOptionsPrefix(_mat, "");
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetFromOptions(_mat);
LIBMESH_CHKERRABORT(ierr);
this->zero();
}
void PetscMatrix<T>::init (const numeric_index_type m_in,
const numeric_index_type n_in,
const numeric_index_type m_l,
const numeric_index_type n_l,
const numeric_index_type nnz,
const numeric_index_type noz,
const numeric_index_type blocksize_in)
{
// So compilers don't warn when !LIBMESH_ENABLE_BLOCKED_STORAGE
libmesh_ignore(blocksize_in);
// Clear initialized matrices
if (this->initialized())
this->clear();
this->_is_initialized = true;
PetscErrorCode ierr = 0;
PetscInt m_global = static_cast<PetscInt>(m_in);
PetscInt n_global = static_cast<PetscInt>(n_in);
PetscInt m_local = static_cast<PetscInt>(m_l);
PetscInt n_local = static_cast<PetscInt>(n_l);
PetscInt n_nz = static_cast<PetscInt>(nnz);
PetscInt n_oz = static_cast<PetscInt>(noz);
ierr = MatCreate(this->comm().get(), &_mat);
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetSizes(_mat, m_local, n_local, m_global, n_global);
LIBMESH_CHKERRABORT(ierr);
#ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
PetscInt blocksize = static_cast<PetscInt>(blocksize_in);
if (blocksize > 1)
{
// specified blocksize, bs>1.
// double check sizes.
libmesh_assert_equal_to (m_local % blocksize, 0);
libmesh_assert_equal_to (n_local % blocksize, 0);
libmesh_assert_equal_to (m_global % blocksize, 0);
libmesh_assert_equal_to (n_global % blocksize, 0);
libmesh_assert_equal_to (n_nz % blocksize, 0);
libmesh_assert_equal_to (n_oz % blocksize, 0);
ierr = MatSetType(_mat, MATBAIJ); // Automatically chooses seqbaij or mpibaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetBlockSize(_mat, blocksize);
LIBMESH_CHKERRABORT(ierr);
ierr = MatSeqBAIJSetPreallocation(_mat, blocksize, n_nz/blocksize, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIBAIJSetPreallocation(_mat, blocksize,
n_nz/blocksize, PETSC_NULL,
n_oz/blocksize, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
}
else
#endif
{
ierr = MatSetType(_mat, MATAIJ); // Automatically chooses seqaij or mpiaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSeqAIJSetPreallocation(_mat, n_nz, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIAIJSetPreallocation(_mat, n_nz, PETSC_NULL, n_oz, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
}
// Make it an error for PETSc to allocate new nonzero entries during assembly
#if PETSC_VERSION_LESS_THAN(3,0,0)
ierr = MatSetOption(_mat, MAT_NEW_NONZERO_ALLOCATION_ERR);
#else
ierr = MatSetOption(_mat, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE);
#endif
LIBMESH_CHKERRABORT(ierr);
// Is prefix information available somewhere? Perhaps pass in the system name?
ierr = MatSetOptionsPrefix(_mat, "");
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetFromOptions(_mat);
LIBMESH_CHKERRABORT(ierr);
this->zero ();
}
PetscErrorCode MatGetMultiProcBlock_MPIBAIJ(Mat mat, MPI_Comm subComm, MatReuse scall,Mat *subMat)
{
PetscErrorCode ierr;
Mat_MPIBAIJ *aij = (Mat_MPIBAIJ*)mat->data;
Mat_SeqBAIJ *aijB = (Mat_SeqBAIJ*)aij->B->data;
PetscMPIInt commRank,subCommSize,subCommRank;
PetscMPIInt *commRankMap,subRank,rank,commsize;
PetscInt *garrayCMap,col,i,j,*nnz,newRow,newCol,*newbRow,*newbCol,k,k1;
PetscInt bs=mat->rmap->bs;
PetscScalar *vals,*aijBvals;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&commsize);CHKERRQ(ierr);
ierr = MPI_Comm_size(subComm,&subCommSize);CHKERRQ(ierr);
/* create subMat object with the relavent layout */
if (scall == MAT_INITIAL_MATRIX) {
ierr = MatCreate(subComm,subMat);CHKERRQ(ierr);
ierr = MatSetType(*subMat,MATMPIBAIJ);CHKERRQ(ierr);
ierr = MatSetSizes(*subMat,mat->rmap->n,mat->cmap->n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetBlockSizes(*subMat,mat->rmap->bs,mat->cmap->bs);CHKERRQ(ierr);
/* need to setup rmap and cmap before Preallocation */
ierr = PetscLayoutSetBlockSize((*subMat)->rmap,mat->rmap->bs);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize((*subMat)->cmap,mat->cmap->bs);CHKERRQ(ierr);
ierr = PetscLayoutSetUp((*subMat)->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp((*subMat)->cmap);CHKERRQ(ierr);
}
/* create a map of comm_rank from subComm to comm - should commRankMap and garrayCMap be kept for reused? */
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)mat),&commRank);CHKERRQ(ierr);
ierr = MPI_Comm_rank(subComm,&subCommRank);CHKERRQ(ierr);
ierr = PetscMalloc(subCommSize*sizeof(PetscMPIInt),&commRankMap);CHKERRQ(ierr);
ierr = MPI_Allgather(&commRank,1,MPI_INT,commRankMap,1,MPI_INT,subComm);CHKERRQ(ierr);
/* Traverse garray and identify blocked column indices [of offdiag mat] that
should be discarded. For the ones not discarded, store the newCol+1
value in garrayCMap */
ierr = PetscMalloc(aij->B->cmap->n/bs*sizeof(PetscInt),&garrayCMap);CHKERRQ(ierr);
ierr = PetscMemzero(garrayCMap,aij->B->cmap->n/bs*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0; i<aij->B->cmap->n/bs; i++) {
col = aij->garray[i]; /* blocked column index */
for (subRank=0; subRank<subCommSize; subRank++) {
rank = commRankMap[subRank];
if ((col >= mat->cmap->range[rank]/bs) && (col < mat->cmap->range[rank+1]/bs)) {
garrayCMap[i] = (((*subMat)->cmap->range[subRank]- mat->cmap->range[rank])/bs + col + 1);
break;
}
}
}
if (scall == MAT_INITIAL_MATRIX) {
/* Now compute preallocation for the offdiag mat */
ierr = PetscMalloc(aij->B->rmap->n/bs*sizeof(PetscInt),&nnz);CHKERRQ(ierr);
ierr = PetscMemzero(nnz,aij->B->rmap->n/bs*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0; i<aij->B->rmap->n/bs; i++) {
for (j=aijB->i[i]; j<aijB->i[i+1]; j++) {
if (garrayCMap[aijB->j[j]]) nnz[i]++;
}
}
ierr = MatMPIBAIJSetPreallocation(*(subMat),bs,0,NULL,0,nnz);CHKERRQ(ierr);
/* reuse diag block with the new submat */
ierr = MatDestroy(&((Mat_MPIBAIJ*)((*subMat)->data))->A);CHKERRQ(ierr);
((Mat_MPIBAIJ*)((*subMat)->data))->A = aij->A;
ierr = PetscObjectReference((PetscObject)aij->A);CHKERRQ(ierr);
} else if (((Mat_MPIBAIJ*)(*subMat)->data)->A != aij->A) {
PetscObject obj = (PetscObject)((Mat_MPIBAIJ*)((*subMat)->data))->A;
ierr = PetscObjectReference((PetscObject)obj);CHKERRQ(ierr);
((Mat_MPIBAIJ*)((*subMat)->data))->A = aij->A;
ierr = PetscObjectReference((PetscObject)aij->A);CHKERRQ(ierr);
}
/* Now traverse aij->B and insert values into subMat */
ierr = PetscMalloc3(bs,PetscInt,&newbRow,bs,PetscInt,&newbCol,bs*bs,PetscScalar,&vals);CHKERRQ(ierr);
for (i=0; i<aij->B->rmap->n/bs; i++) {
newRow = (*subMat)->rmap->range[subCommRank] + i*bs;
for (j=aijB->i[i]; j<aijB->i[i+1]; j++) {
newCol = garrayCMap[aijB->j[j]];
if (newCol) {
newCol--; /* remove the increment */
newCol *= bs;
for (k=0; k<bs; k++) {
newbRow[k] = newRow + k;
newbCol[k] = newCol + k;
}
/* copy column-oriented aijB->a into row-oriented vals */
aijBvals = aijB->a + j*bs*bs;
for (k1=0; k1<bs; k1++) {
for (k=0; k<bs; k++) {
vals[k1+k*bs] = *aijBvals++;
}
}
ierr = MatSetValues(*subMat,bs,newbRow,bs,newbCol,vals,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(*subMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*subMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* deallocate temporary data */
ierr = PetscFree3(newbRow,newbCol,vals);CHKERRQ(ierr);
ierr = PetscFree(commRankMap);CHKERRQ(ierr);
ierr = PetscFree(garrayCMap);CHKERRQ(ierr);
if (scall == MAT_INITIAL_MATRIX) {
ierr = PetscFree(nnz);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat C,A;
PetscInt i, n = 10,midx[3],bs=1;
PetscErrorCode ierr;
PetscScalar v[3];
PetscBool flg,isAIJ;
MatType type;
PetscMPIInt size;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_block_size",&bs,PETSC_NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetType(C,MATAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatGetType(C,&type);CHKERRQ(ierr);
if (size == 1){
ierr = PetscObjectTypeCompare((PetscObject)C,MATSEQAIJ,&isAIJ);CHKERRQ(ierr);
} else {
ierr = PetscObjectTypeCompare((PetscObject)C,MATMPIAIJ,&isAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocation(C,3,PETSC_NULL);
ierr = MatMPIAIJSetPreallocation(C,3,PETSC_NULL,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(C,bs,3,PETSC_NULL);
ierr = MatMPIBAIJSetPreallocation(C,bs,3,PETSC_NULL,3,PETSC_NULL);CHKERRQ(ierr);
v[0] = -1.; v[1] = 2.; v[2] = -1.;
for (i=1; i<n-1; i++){
midx[2] = i-1; midx[1] = i; midx[0] = i+1;
ierr = MatSetValues(C,1,&i,3,midx,v,INSERT_VALUES);CHKERRQ(ierr);
}
i = 0; midx[0] = 0; midx[1] = 1;
v[0] = 2.0; v[1] = -1.;
ierr = MatSetValues(C,1,&i,2,midx,v,INSERT_VALUES);CHKERRQ(ierr);
i = n-1; midx[0] = n-2; midx[1] = n-1;
v[0] = -1.0; v[1] = 2.;
ierr = MatSetValues(C,1,&i,2,midx,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* test matrices with different nonzero patterns - Note: A is created with different nonzero pattern of C! */
ierr = MatCopy(C,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatEqual(A,C,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"MatCopy(C,A,DIFFERENT_NONZERO_PATTERN): Matrices are NOT equal");
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"A is obtained with MatCopy(,,DIFFERENT_NONZERO_PATTERN):\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
/* test matrices with same nonzero pattern */
ierr = MatDuplicate(C,MAT_DO_NOT_COPY_VALUES,&A);CHKERRQ(ierr);
ierr = MatCopy(C,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatEqual(A,C,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"MatCopy(C,A,SAME_NONZERO_PATTERN): Matrices are NOT equal");
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nA is obtained with MatCopy(,,SAME_NONZERO_PATTERN):\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"A:\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test MatStore/RetrieveValues() */
if (isAIJ){
ierr = MatSetOption(A,MAT_NEW_NONZERO_LOCATIONS,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatStoreValues(A);CHKERRQ(ierr);
ierr = MatZeroEntries(A);CHKERRQ(ierr);
ierr = MatRetrieveValues(A);CHKERRQ(ierr);
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatISGetMPIXAIJ_IS(Mat mat, MatReuse reuse, Mat *M)
{
Mat_IS *matis = (Mat_IS*)(mat->data);
/* info on mat */
/* ISLocalToGlobalMapping rmapping,cmapping; */
PetscInt bs,rows,cols;
PetscInt lrows,lcols;
PetscInt local_rows,local_cols;
PetscBool isdense,issbaij,issbaij_red;
/* values insertion */
PetscScalar *array;
PetscInt *local_indices,*global_indices;
/* work */
PetscInt i,j,index_row;
PetscErrorCode ierr;
PetscFunctionBegin;
/* MISSING CHECKS
- rectangular case not covered (it is not allowed by MATIS)
*/
/* get info from mat */
/* ierr = MatGetLocalToGlobalMapping(mat,&rmapping,&cmapping);CHKERRQ(ierr); */
ierr = MatGetSize(mat,&rows,&cols);CHKERRQ(ierr);
ierr = MatGetBlockSize(mat,&bs);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
/* work */
ierr = PetscMalloc1(local_rows,&local_indices);CHKERRQ(ierr);
for (i=0;i<local_rows;i++) local_indices[i]=i;
/* map indices of local mat to global values */
ierr = PetscMalloc(PetscMax(local_cols,local_rows)*sizeof(*global_indices),&global_indices);CHKERRQ(ierr);
/* ierr = ISLocalToGlobalMappingApply(rmapping,local_rows,local_indices,global_indices);CHKERRQ(ierr); */
ierr = ISLocalToGlobalMappingApply(matis->mapping,local_rows,local_indices,global_indices);CHKERRQ(ierr);
if (issbaij) {
ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
if (reuse == MAT_INITIAL_MATRIX) {
Mat new_mat;
MatType new_mat_type;
Vec vec_dnz,vec_onz;
PetscScalar *my_dnz,*my_onz;
PetscInt *dnz,*onz,*mat_ranges,*row_ownership;
PetscInt index_col,owner;
PetscMPIInt nsubdomains;
/* determining new matrix type */
ierr = MPI_Allreduce(&issbaij,&issbaij_red,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr);
if (issbaij_red) {
new_mat_type = MATSBAIJ;
} else {
if (bs>1) {
new_mat_type = MATBAIJ;
} else {
new_mat_type = MATAIJ;
}
}
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&nsubdomains);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)mat),&new_mat);CHKERRQ(ierr);
ierr = MatSetSizes(new_mat,PETSC_DECIDE,PETSC_DECIDE,rows,cols);CHKERRQ(ierr);
ierr = MatSetBlockSize(new_mat,bs);CHKERRQ(ierr);
ierr = MatSetType(new_mat,new_mat_type);CHKERRQ(ierr);
ierr = MatSetUp(new_mat);CHKERRQ(ierr);
ierr = MatGetLocalSize(new_mat,&lrows,&lcols);CHKERRQ(ierr);
/*
preallocation
*/
ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)new_mat),lrows,lcols,dnz,onz);CHKERRQ(ierr);
/*
Some vectors are needed to sum up properly on shared interface dofs.
Preallocation macros cannot do the job.
Note that preallocation is not exact, since it overestimates nonzeros
*/
ierr = MatCreateVecs(new_mat,NULL,&vec_dnz);CHKERRQ(ierr);
/* ierr = VecSetLocalToGlobalMapping(vec_dnz,rmapping);CHKERRQ(ierr); */
ierr = VecSetLocalToGlobalMapping(vec_dnz,matis->mapping);CHKERRQ(ierr);
ierr = VecDuplicate(vec_dnz,&vec_onz);CHKERRQ(ierr);
/* All processes need to compute entire row ownership */
ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(new_mat,(const PetscInt**)&mat_ranges);CHKERRQ(ierr);
for (i=0;i<nsubdomains;i++) {
for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
row_ownership[j]=i;
}
}
/*
my_dnz and my_onz contains exact contribution to preallocation from each local mat
then, they will be summed up properly. This way, preallocation is always sufficient
*/
ierr = PetscMalloc1(local_rows,&my_dnz);CHKERRQ(ierr);
ierr = PetscMalloc1(local_rows,&my_onz);CHKERRQ(ierr);
ierr = PetscMemzero(my_dnz,local_rows*sizeof(*my_dnz));CHKERRQ(ierr);
ierr = PetscMemzero(my_onz,local_rows*sizeof(*my_onz));CHKERRQ(ierr);
/* preallocation as a MATAIJ */
if (isdense) { /* special case for dense local matrices */
for (i=0;i<local_rows;i++) {
index_row = global_indices[i];
for (j=i;j<local_rows;j++) {
owner = row_ownership[index_row];
index_col = global_indices[j];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1.0;
} else { /* offdiag block */
my_onz[i] += 1.0;
}
/* same as before, interchanging rows and cols */
if (i != j) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1.0;
} else {
my_onz[j] += 1.0;
}
}
}
}
} else {
for (i=0;i<local_rows;i++) {
PetscInt ncols;
const PetscInt *cols;
index_row = global_indices[i];
ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
owner = row_ownership[index_row];
index_col = global_indices[cols[j]];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1.0;
} else { /* offdiag block */
my_onz[i] += 1.0;
}
/* same as before, interchanging rows and cols */
if (issbaij) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1.0;
} else {
my_onz[j] += 1.0;
}
}
}
ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
}
}
ierr = VecSet(vec_dnz,0.0);CHKERRQ(ierr);
ierr = VecSet(vec_onz,0.0);CHKERRQ(ierr);
if (local_rows) { /* multilevel guard */
ierr = VecSetValuesLocal(vec_dnz,local_rows,local_indices,my_dnz,ADD_VALUES);CHKERRQ(ierr);
ierr = VecSetValuesLocal(vec_onz,local_rows,local_indices,my_onz,ADD_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(vec_dnz);CHKERRQ(ierr);
ierr = VecAssemblyBegin(vec_onz);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec_dnz);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec_onz);CHKERRQ(ierr);
ierr = PetscFree(my_dnz);CHKERRQ(ierr);
ierr = PetscFree(my_onz);CHKERRQ(ierr);
ierr = PetscFree(row_ownership);CHKERRQ(ierr);
/* set computed preallocation in dnz and onz */
ierr = VecGetArray(vec_dnz,&array);CHKERRQ(ierr);
for (i=0; i<lrows; i++) dnz[i] = (PetscInt)PetscRealPart(array[i]);
ierr = VecRestoreArray(vec_dnz,&array);CHKERRQ(ierr);
ierr = VecGetArray(vec_onz,&array);CHKERRQ(ierr);
for (i=0;i<lrows;i++) onz[i] = (PetscInt)PetscRealPart(array[i]);
ierr = VecRestoreArray(vec_onz,&array);CHKERRQ(ierr);
ierr = VecDestroy(&vec_dnz);CHKERRQ(ierr);
ierr = VecDestroy(&vec_onz);CHKERRQ(ierr);
/* Resize preallocation if overestimated */
for (i=0;i<lrows;i++) {
dnz[i] = PetscMin(dnz[i],lcols);
onz[i] = PetscMin(onz[i],cols-lcols);
}
/* set preallocation */
ierr = MatMPIAIJSetPreallocation(new_mat,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i*bs]/bs;
onz[i] = onz[i*bs]/bs;
}
ierr = MatMPIBAIJSetPreallocation(new_mat,bs,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i]-i;
}
ierr = MatMPISBAIJSetPreallocation(new_mat,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr);
*M = new_mat;
} else {
PetscInt mbs,mrows,mcols;
/* some checks */
ierr = MatGetBlockSize(*M,&mbs);CHKERRQ(ierr);
ierr = MatGetSize(*M,&mrows,&mcols);CHKERRQ(ierr);
if (mrows != rows) {
SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of rows (%d != %d)",rows,mrows);
}
if (mrows != rows) {
SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of cols (%d != %d)",cols,mcols);
}
if (mbs != bs) {
SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong block size (%d != %d)",bs,mbs);
}
ierr = MatZeroEntries(*M);CHKERRQ(ierr);
}
/* set local to global mappings */
/* ierr = MatSetLocalToGlobalMapping(*M,rmapping,cmapping);CHKERRQ(ierr); */
/* Set values */
if (isdense) { /* special case for dense local matrices */
ierr = MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatDenseGetArray(matis->A,&array);CHKERRQ(ierr);
ierr = MatSetValues(*M,local_rows,global_indices,local_cols,global_indices,array,ADD_VALUES);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(matis->A,&array);CHKERRQ(ierr);
ierr = PetscFree(local_indices);CHKERRQ(ierr);
ierr = PetscFree(global_indices);CHKERRQ(ierr);
} else { /* very basic values insertion for all other matrix types */
ierr = PetscFree(local_indices);CHKERRQ(ierr);
for (i=0;i<local_rows;i++) {
ierr = MatGetRow(matis->A,i,&j,(const PetscInt**)&local_indices,(const PetscScalar**)&array);CHKERRQ(ierr);
/* ierr = MatSetValuesLocal(*M,1,&i,j,local_indices,array,ADD_VALUES);CHKERRQ(ierr); */
ierr = ISLocalToGlobalMappingApply(matis->mapping,j,local_indices,global_indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(matis->mapping,1,&i,&index_row);CHKERRQ(ierr);
ierr = MatSetValues(*M,1,&index_row,j,global_indices,array,ADD_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(matis->A,i,&j,(const PetscInt**)&local_indices,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = PetscFree(global_indices);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (isdense) {
ierr = MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_TRUE);CHKERRQ(ierr);
}
if (issbaij) {
ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
