PetscErrorCode DMSetUp_Composite(DM dm)
{
PetscErrorCode ierr;
PetscInt nprev = 0;
PetscMPIInt rank,size;
DM_Composite *com = (DM_Composite*)dm->data;
struct DMCompositeLink *next = com->next;
PetscLayout map;
PetscFunctionBegin;
if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup");
ierr = PetscLayoutCreate(((PetscObject)dm)->comm,&map);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(map,com->n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(map,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(map);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(map,&com->N);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr);
/* now set the rstart for each linked vector */
ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)dm)->comm,&size);CHKERRQ(ierr);
while (next) {
next->rstart = nprev;
nprev += next->n;
next->grstart = com->rstart + next->rstart;
ierr = PetscMalloc(size*sizeof(PetscInt),&next->grstarts);CHKERRQ(ierr);
ierr = MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);CHKERRQ(ierr);
next = next->next;
}
com->setup = PETSC_TRUE;
PetscFunctionReturn(0);
}
/*@C
VecCreateNest - Creates a new vector containing several nested subvectors, each stored separately
Collective on Vec
Input Parameter:
+ comm - Communicator for the new Vec
. nb - number of nested blocks
. is - array of nb index sets describing each nested block, or NULL to pack subvectors contiguously
- x - array of nb sub-vectors
Output Parameter:
. Y - new vector
Level: advanced
.seealso: VecCreate(), MatCreateNest(), DMSetVecType(), VECNEST
@*/
PetscErrorCode VecCreateNest(MPI_Comm comm,PetscInt nb,IS is[],Vec x[],Vec *Y)
{
Vec V;
Vec_Nest *s;
PetscInt n,N;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = VecCreate(comm,&V);CHKERRQ(ierr);
ierr = PetscObjectChangeTypeName((PetscObject)V,VECNEST);CHKERRQ(ierr);
/* allocate and set pointer for implememtation data */
ierr = PetscMalloc(sizeof(Vec_Nest),&s);CHKERRQ(ierr);
ierr = PetscMemzero(s,sizeof(Vec_Nest));CHKERRQ(ierr);
V->data = (void*)s;
s->setup_called = PETSC_FALSE;
s->nb = -1;
s->v = NULL;
ierr = VecSetUp_Nest_Private(V,nb,x);CHKERRQ(ierr);
n = N = 0;
ierr = VecSize_Nest_Recursive(V,PETSC_TRUE,&N);CHKERRQ(ierr);
ierr = VecSize_Nest_Recursive(V,PETSC_FALSE,&n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(V->map,N);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(V->map,n);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(V->map,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(V->map);CHKERRQ(ierr);
ierr = VecSetUp_NestIS_Private(V,nb,is);CHKERRQ(ierr);
ierr = VecNestSetOps_Private(V->ops);CHKERRQ(ierr);
V->petscnative = PETSC_FALSE;
/* expose block api's */
ierr = PetscObjectComposeFunction((PetscObject)V,"VecNestGetSubVec_C",VecNestGetSubVec_Nest);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)V,"VecNestGetSubVecs_C",VecNestGetSubVecs_Nest);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)V,"VecNestSetSubVec_C",VecNestSetSubVec_Nest);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)V,"VecNestSetSubVecs_C",VecNestSetSubVecs_Nest);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)V,"VecNestGetSize_C",VecNestGetSize_Nest);CHKERRQ(ierr);
*Y = V;
PetscFunctionReturn(0);
}
static PetscErrorCode ISSortRemoveDups_General(IS is)
{
IS_General *sub = (IS_General*)is->data;
PetscInt n;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
if (sub->sorted) {
ierr = PetscSortedRemoveDupsInt(&n,sub->idx);CHKERRQ(ierr);
} else {
ierr = PetscSortRemoveDupsInt(&n,sub->idx);CHKERRQ(ierr);
}
ierr = PetscLayoutSetLocalSize(is->map, n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(is->map, PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(is->map);CHKERRQ(ierr);
sub->sorted = PETSC_TRUE;
PetscFunctionReturn(0);
}
static PetscErrorCode ISSortRemoveDups_Block(IS is)
{
IS_Block *sub = (IS_Block*)is->data;
PetscInt bs, n, nb;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetBlockSize(is->map, &bs);CHKERRQ(ierr);
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
nb = n/bs;
if (sub->sorted) {
ierr = PetscSortedRemoveDupsInt(&nb,sub->idx);CHKERRQ(ierr);
} else {
ierr = PetscSortRemoveDupsInt(&nb,sub->idx);CHKERRQ(ierr);
}
ierr = PetscLayoutSetLocalSize(is->map, nb*bs);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(is->map, PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(is->map);CHKERRQ(ierr);
sub->sorted = PETSC_TRUE;
PetscFunctionReturn(0);
}
static PetscErrorCode VecNestSetSubVec_Private(Vec X,PetscInt idxm,Vec x)
{
Vec_Nest *bx = (Vec_Nest*)X->data;
PetscInt i,offset=0,n=0,bs;
IS is;
PetscErrorCode ierr;
PetscBool issame = PETSC_FALSE;
PetscInt N=0;
/* check if idxm < bx->nb */
if (idxm >= bx->nb) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %D maximum %D",idxm,bx->nb);
PetscFunctionBegin;
ierr = VecDestroy(&bx->v[idxm]);CHKERRQ(ierr); /* destroy the existing vector */
ierr = VecDuplicate(x,&bx->v[idxm]);CHKERRQ(ierr); /* duplicate the layout of given vector */
ierr = VecCopy(x,bx->v[idxm]);CHKERRQ(ierr); /* copy the contents of the given vector */
/* check if we need to update the IS for the block */
offset = X->map->rstart;
for (i=0; i<idxm; i++) {
n=0;
ierr = VecGetLocalSize(bx->v[i],&n);CHKERRQ(ierr);
offset += n;
}
/* get the local size and block size */
ierr = VecGetLocalSize(x,&n);CHKERRQ(ierr);
ierr = VecGetBlockSize(x,&bs);CHKERRQ(ierr);
/* create the new IS */
ierr = ISCreateStride(PetscObjectComm((PetscObject)x),n,offset,1,&is);CHKERRQ(ierr);
ierr = ISSetBlockSize(is,bs);CHKERRQ(ierr);
/* check if they are equal */
ierr = ISEqual(is,bx->is[idxm],&issame);CHKERRQ(ierr);
if (!issame) {
/* The IS of given vector has a different layout compared to the existing block vector.
Destroy the existing reference and update the IS. */
ierr = ISDestroy(&bx->is[idxm]);CHKERRQ(ierr);
ierr = ISDuplicate(is,&bx->is[idxm]);CHKERRQ(ierr);
ierr = ISCopy(is,bx->is[idxm]);CHKERRQ(ierr);
offset += n;
/* Since the current IS[idxm] changed, we need to update all the subsequent IS */
for (i=idxm+1; i<bx->nb; i++) {
/* get the local size and block size */
ierr = VecGetLocalSize(bx->v[i],&n);CHKERRQ(ierr);
ierr = VecGetBlockSize(bx->v[i],&bs);CHKERRQ(ierr);
/* destroy the old and create the new IS */
ierr = ISDestroy(&bx->is[i]);CHKERRQ(ierr);
ierr = ISCreateStride(((PetscObject)bx->v[i])->comm,n,offset,1,&bx->is[i]);CHKERRQ(ierr);
ierr = ISSetBlockSize(bx->is[i],bs);CHKERRQ(ierr);
offset += n;
}
n=0;
ierr = VecSize_Nest_Recursive(X,PETSC_TRUE,&N);CHKERRQ(ierr);
ierr = VecSize_Nest_Recursive(X,PETSC_FALSE,&n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(X->map,N);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(X->map,n);CHKERRQ(ierr);
}
ierr = ISDestroy(&is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_Redistribute(PC pc)
{
PC_Redistribute *red = (PC_Redistribute*)pc->data;
PetscErrorCode ierr;
MPI_Comm comm;
PetscInt rstart,rend,i,nz,cnt,*rows,ncnt,dcnt,*drows;
PetscLayout map,nmap;
PetscMPIInt size,imdex,tag,n;
PetscInt *source = PETSC_NULL;
PetscMPIInt *nprocs = PETSC_NULL,nrecvs;
PetscInt j,nsends;
PetscInt *owner = PETSC_NULL,*starts = PETSC_NULL,count,slen;
PetscInt *rvalues,*svalues,recvtotal;
PetscMPIInt *onodes1,*olengths1;
MPI_Request *send_waits = PETSC_NULL,*recv_waits = PETSC_NULL;
MPI_Status recv_status,*send_status;
Vec tvec,diag;
Mat tmat;
const PetscScalar *d;
PetscFunctionBegin;
if (pc->setupcalled) {
ierr = KSPGetOperators(red->ksp,PETSC_NULL,&tmat,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->pmat,red->is,red->is,MAT_REUSE_MATRIX,&tmat);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,tmat,tmat,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
} else {
PetscInt NN;
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscObjectGetNewTag((PetscObject)pc,&tag);CHKERRQ(ierr);
/* count non-diagonal rows on process */
ierr = MatGetOwnershipRange(pc->mat,&rstart,&rend);CHKERRQ(ierr);
cnt = 0;
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
if (nz > 1) cnt++;
ierr = MatRestoreRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
}
ierr = PetscMalloc(cnt*sizeof(PetscInt),&rows);CHKERRQ(ierr);
ierr = PetscMalloc((rend - rstart - cnt)*sizeof(PetscInt),&drows);CHKERRQ(ierr);
/* list non-diagonal rows on process */
cnt = 0; dcnt = 0;
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
if (nz > 1) rows[cnt++] = i;
else drows[dcnt++] = i - rstart;
ierr = MatRestoreRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
}
/* create PetscLayout for non-diagonal rows on each process */
ierr = PetscLayoutCreate(comm,&map);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(map,cnt);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(map);CHKERRQ(ierr);
rstart = map->rstart;
rend = map->rend;
/* create PetscLayout for load-balanced non-diagonal rows on each process */
ierr = PetscLayoutCreate(comm,&nmap);CHKERRQ(ierr);
ierr = MPI_Allreduce(&cnt,&ncnt,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(nmap,ncnt);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(nmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(nmap);CHKERRQ(ierr);
ierr = MatGetSize(pc->pmat,&NN,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscInfo2(pc,"Number of diagonal rows eliminated %d, percentage eliminated %g\n",NN-ncnt,((PetscReal)(NN-ncnt))/((PetscReal)(NN)));CHKERRQ(ierr);
/*
this code is taken from VecScatterCreate_PtoS()
Determines what rows need to be moved where to
load balance the non-diagonal rows
*/
/* count number of contributors to each processor */
ierr = PetscMalloc2(size,PetscMPIInt,&nprocs,cnt,PetscInt,&owner);CHKERRQ(ierr);
ierr = PetscMemzero(nprocs,size*sizeof(PetscMPIInt));CHKERRQ(ierr);
j = 0;
nsends = 0;
for (i=rstart; i<rend; i++) {
if (i < nmap->range[j]) j = 0;
for (; j<size; j++) {
if (i < nmap->range[j+1]) {
if (!nprocs[j]++) nsends++;
owner[i-rstart] = j;
break;
}
}
}
/* inform other processors of number of messages and max length*/
ierr = PetscGatherNumberOfMessages(comm,PETSC_NULL,nprocs,&nrecvs);CHKERRQ(ierr);
ierr = PetscGatherMessageLengths(comm,nsends,nrecvs,nprocs,&onodes1,&olengths1);CHKERRQ(ierr);
ierr = PetscSortMPIIntWithArray(nrecvs,onodes1,olengths1);CHKERRQ(ierr);
recvtotal = 0; for (i=0; i<nrecvs; i++) recvtotal += olengths1[i];
/* post receives: rvalues - rows I will own; count - nu */
ierr = PetscMalloc3(recvtotal,PetscInt,&rvalues,nrecvs,PetscInt,&source,nrecvs,MPI_Request,&recv_waits);CHKERRQ(ierr);
count = 0;
for (i=0; i<nrecvs; i++) {
ierr = MPI_Irecv((rvalues+count),olengths1[i],MPIU_INT,onodes1[i],tag,comm,recv_waits+i);CHKERRQ(ierr);
count += olengths1[i];
}
/* do sends:
1) starts[i] gives the starting index in svalues for stuff going to
the ith processor
*/
ierr = PetscMalloc3(cnt,PetscInt,&svalues,nsends,MPI_Request,&send_waits,size,PetscInt,&starts);CHKERRQ(ierr);
starts[0] = 0;
for (i=1; i<size; i++) { starts[i] = starts[i-1] + nprocs[i-1];}
for (i=0; i<cnt; i++) {
svalues[starts[owner[i]]++] = rows[i];
}
for (i=0; i<cnt; i++) rows[i] = rows[i] - rstart;
red->drows = drows;
red->dcnt = dcnt;
ierr = PetscFree(rows);CHKERRQ(ierr);
starts[0] = 0;
for (i=1; i<size; i++) { starts[i] = starts[i-1] + nprocs[i-1];}
count = 0;
for (i=0; i<size; i++) {
if (nprocs[i]) {
ierr = MPI_Isend(svalues+starts[i],nprocs[i],MPIU_INT,i,tag,comm,send_waits+count++);CHKERRQ(ierr);
}
}
/* wait on receives */
count = nrecvs;
slen = 0;
while (count) {
ierr = MPI_Waitany(nrecvs,recv_waits,&imdex,&recv_status);CHKERRQ(ierr);
/* unpack receives into our local space */
ierr = MPI_Get_count(&recv_status,MPIU_INT,&n);CHKERRQ(ierr);
slen += n;
count--;
}
if (slen != recvtotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Total message lengths %D not expected %D",slen,recvtotal);
ierr = ISCreateGeneral(comm,slen,rvalues,PETSC_COPY_VALUES,&red->is);CHKERRQ(ierr);
/* free up all work space */
ierr = PetscFree(olengths1);CHKERRQ(ierr);
ierr = PetscFree(onodes1);CHKERRQ(ierr);
ierr = PetscFree3(rvalues,source,recv_waits);CHKERRQ(ierr);
ierr = PetscFree2(nprocs,owner);CHKERRQ(ierr);
if (nsends) { /* wait on sends */
ierr = PetscMalloc(nsends*sizeof(MPI_Status),&send_status);CHKERRQ(ierr);
ierr = MPI_Waitall(nsends,send_waits,send_status);CHKERRQ(ierr);
ierr = PetscFree(send_status);CHKERRQ(ierr);
}
ierr = PetscFree3(svalues,send_waits,starts);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&nmap);CHKERRQ(ierr);
ierr = VecCreateMPI(comm,slen,PETSC_DETERMINE,&red->b);CHKERRQ(ierr);
ierr = VecDuplicate(red->b,&red->x);CHKERRQ(ierr);
ierr = MatGetVecs(pc->pmat,&tvec,PETSC_NULL);CHKERRQ(ierr);
ierr = VecScatterCreate(tvec,red->is,red->b,PETSC_NULL,&red->scatter);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->pmat,red->is,red->is,MAT_INITIAL_MATRIX,&tmat);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,tmat,tmat,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&tmat);CHKERRQ(ierr);
}
/* get diagonal portion of matrix */
ierr = PetscMalloc(red->dcnt*sizeof(PetscScalar),&red->diag);CHKERRQ(ierr);
ierr = MatGetVecs(pc->pmat,&diag,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetDiagonal(pc->pmat,diag);CHKERRQ(ierr);
ierr = VecGetArrayRead(diag,&d);CHKERRQ(ierr);
for (i=0; i<red->dcnt; i++) {
red->diag[i] = 1.0/d[red->drows[i]];
}
ierr = VecRestoreArrayRead(diag,&d);CHKERRQ(ierr);
ierr = VecDestroy(&diag);CHKERRQ(ierr);
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatLoad_AIJ_HDF5(Mat mat, PetscViewer viewer)
{
PetscViewerFormat format;
const PetscInt *i_glob = NULL;
PetscInt *i = NULL;
const PetscInt *j = NULL;
const PetscScalar *a = NULL;
const char *a_name = NULL, *i_name = NULL, *j_name = NULL, *mat_name = NULL, *c_name = NULL;
PetscInt p, m, M, N;
PetscInt bs = mat->rmap->bs;
PetscBool flg;
IS is_i = NULL, is_j = NULL;
Vec vec_a = NULL;
PetscLayout jmap = NULL;
MPI_Comm comm;
PetscMPIInt rank, size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
switch (format) {
case PETSC_VIEWER_HDF5_PETSC:
case PETSC_VIEWER_DEFAULT:
case PETSC_VIEWER_NATIVE:
case PETSC_VIEWER_HDF5_MAT:
break;
default:
SETERRQ1(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"PetscViewerFormat %s not supported for HDF5 input.",PetscViewerFormats[format]);
}
ierr = PetscObjectGetComm((PetscObject)mat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)mat,&mat_name);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetAIJNames(viewer,&i_name,&j_name,&a_name,&c_name);CHKERRQ(ierr);
ierr = PetscOptionsBegin(comm,NULL,"Options for loading matrix from HDF5","Mat");CHKERRQ(ierr);
ierr = PetscOptionsInt("-matload_block_size","Set the blocksize used to store the matrix","MatLoad",bs,&bs,&flg);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
if (flg) {
ierr = MatSetBlockSize(mat, bs);CHKERRQ(ierr);
}
ierr = PetscViewerHDF5PushGroup(viewer,mat_name);CHKERRQ(ierr);
ierr = PetscViewerHDF5ReadAttribute(viewer,NULL,c_name,PETSC_INT,&N);CHKERRQ(ierr);
ierr = PetscViewerHDF5ReadSizes(viewer, i_name, NULL, &M);CHKERRQ(ierr);
--M; /* i has size M+1 as there is global number of nonzeros stored at the end */
if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) {
/* Swap row and columns layout for unallocated matrix. I want to avoid calling MatTranspose() just to transpose sparsity pattern and layout. */
if (!mat->preallocated) {
PetscLayout tmp;
tmp = mat->rmap; mat->rmap = mat->cmap; mat->cmap = tmp;
} else SETERRQ(comm,PETSC_ERR_SUP,"Not for preallocated matrix - we would need to transpose it here which we want to avoid");
}
/* If global sizes are set, check if they are consistent with that given in the file */
if (mat->rmap->N >= 0 && mat->rmap->N != M) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of rows: Matrix in file has (%D) and input matrix has (%D)",mat->rmap->N,M);
if (mat->cmap->N >= 0 && mat->cmap->N != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of cols: Matrix in file has (%D) and input matrix has (%D)",mat->cmap->N,N);
/* Determine ownership of all (block) rows and columns */
mat->rmap->N = M;
mat->cmap->N = N;
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
m = mat->rmap->n;
/* Read array i (array of row indices) */
ierr = PetscMalloc1(m+1, &i);CHKERRQ(ierr); /* allocate i with one more position for local number of nonzeros on each rank */
i[0] = i[m] = 0; /* make the last entry always defined - the code block below overwrites it just on last rank */
if (rank == size-1) m++; /* in the loaded array i_glob, only the last rank has one more position with the global number of nonzeros */
M++;
ierr = ISCreate(comm,&is_i);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)is_i,i_name);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(is_i->map,m);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(is_i->map,M);CHKERRQ(ierr);
ierr = ISLoad(is_i,viewer);CHKERRQ(ierr);
ierr = ISGetIndices(is_i,&i_glob);CHKERRQ(ierr);
ierr = PetscMemcpy(i,i_glob,m*sizeof(PetscInt));CHKERRQ(ierr);
/* Reset m and M to the matrix sizes */
m = mat->rmap->n;
M--;
/* Create PetscLayout for j and a vectors; construct ranges first */
ierr = PetscLayoutCreate(comm,&jmap);CHKERRQ(ierr);
ierr = PetscCalloc1(size+1, &jmap->range);CHKERRQ(ierr);
ierr = MPI_Allgather(&i[0], 1, MPIU_INT, jmap->range, 1, MPIU_INT, comm);CHKERRQ(ierr);
/* Last rank has global number of nonzeros (= length of j and a arrays) in i[m] (last i entry) so broadcast it */
jmap->range[size] = i[m];
ierr = MPI_Bcast(&jmap->range[size], 1, MPIU_INT, size-1, comm);CHKERRQ(ierr);
for (p=size-1; p>0; p--) {
if (!jmap->range[p]) jmap->range[p] = jmap->range[p+1]; /* for ranks with 0 rows, take the value from the next processor */
}
i[m] = jmap->range[rank+1]; /* i[m] (last i entry) is equal to next rank's offset */
/* Deduce rstart, rend, n and N from the ranges */
ierr = PetscLayoutSetUp(jmap);CHKERRQ(ierr);
/* Convert global to local indexing of rows */
for (p=1; p<m+1; ++p) i[p] -= i[0];
i[0] = 0;
/* Read array j (array of column indices) */
ierr = ISCreate(comm,&is_j);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)is_j,j_name);CHKERRQ(ierr);
ierr = PetscLayoutDuplicate(jmap,&is_j->map);CHKERRQ(ierr);
ierr = ISLoad(is_j,viewer);CHKERRQ(ierr);
ierr = ISGetIndices(is_j,&j);CHKERRQ(ierr);
/* Read array a (array of values) */
ierr = VecCreate(comm,&vec_a);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)vec_a,a_name);CHKERRQ(ierr);
ierr = PetscLayoutDuplicate(jmap,&vec_a->map);CHKERRQ(ierr);
ierr = VecLoad(vec_a,viewer);CHKERRQ(ierr);
ierr = VecGetArrayRead(vec_a,&a);CHKERRQ(ierr);
/* populate matrix */
if (!((PetscObject)mat)->type_name) {
ierr = MatSetType(mat,MATAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr);
/*
ierr = MatSeqBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr);
*/
if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) {
/* Transpose the input matrix back */
ierr = MatTranspose(mat,MAT_INPLACE_MATRIX,&mat);CHKERRQ(ierr);
}
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&jmap);CHKERRQ(ierr);
ierr = PetscFree(i);CHKERRQ(ierr);
ierr = ISRestoreIndices(is_i,&i_glob);CHKERRQ(ierr);
ierr = ISRestoreIndices(is_j,&j);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(vec_a,&a);CHKERRQ(ierr);
ierr = ISDestroy(&is_i);CHKERRQ(ierr);
ierr = ISDestroy(&is_j);CHKERRQ(ierr);
ierr = VecDestroy(&vec_a);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
