PETSC_EXTERN PetscErrorCode MatHeaderReplace(Mat A,Mat C) { PetscErrorCode ierr; PetscInt refct; PetscFunctionBegin; PetscValidHeaderSpecific(A,MAT_CLASSID,1); PetscValidHeaderSpecific(C,MAT_CLASSID,2); if (A == C) PetscFunctionReturn(0); PetscCheckSameComm(A,1,C,2); if (((PetscObject)C)->refct != 1) SETERRQ1(PetscObjectComm((PetscObject)C),PETSC_ERR_ARG_WRONGSTATE,"Object C has refct %D > 1, would leave hanging reference",((PetscObject)C)->refct); /* free all the interior data structures from mat */ ierr = (*A->ops->destroy)(A);CHKERRQ(ierr); ierr = PetscHeaderDestroy_Private((PetscObject)A);CHKERRQ(ierr); ierr = PetscFree(A->ops);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&A->rmap);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&A->cmap);CHKERRQ(ierr); ierr = PetscFree(A->spptr);CHKERRQ(ierr); /* copy C over to A */ refct = ((PetscObject)A)->refct; ierr = PetscMemcpy(A,C,sizeof(struct _p_Mat));CHKERRQ(ierr); ((PetscObject)A)->refct = refct; ierr = PetscFree(C);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode MatHeaderMerge(Mat A,Mat C) { PetscErrorCode ierr; PetscInt refct; PetscOps *Abops; MatOps Aops; char *mtype,*mname; void *spptr; PetscFunctionBegin; /* save the parts of A we need */ Abops = ((PetscObject)A)->bops; Aops = A->ops; refct = ((PetscObject)A)->refct; mtype = ((PetscObject)A)->type_name; mname = ((PetscObject)A)->name; spptr = A->spptr; /* zero these so the destroy below does not free them */ ((PetscObject)A)->type_name = 0; ((PetscObject)A)->name = 0; /* free all the interior data structures from mat */ ierr = (*A->ops->destroy)(A); CHKERRQ(ierr); ierr = PetscFree(C->spptr); CHKERRQ(ierr); ierr = PetscLayoutDestroy(&A->rmap); CHKERRQ(ierr); ierr = PetscLayoutDestroy(&A->cmap); CHKERRQ(ierr); ierr = PetscFunctionListDestroy(&((PetscObject)A)->qlist); CHKERRQ(ierr); ierr = PetscObjectListDestroy(&((PetscObject)A)->olist); CHKERRQ(ierr); /* copy C over to A */ ierr = PetscMemcpy(A,C,sizeof(struct _p_Mat)); CHKERRQ(ierr); /* return the parts of A we saved */ ((PetscObject)A)->bops = Abops; A->ops = Aops; ((PetscObject)A)->refct = refct; ((PetscObject)A)->type_name = mtype; ((PetscObject)A)->name = mname; A->spptr = spptr; /* since these two are copied into A we do not want them destroyed in C */ ((PetscObject)C)->qlist = 0; ((PetscObject)C)->olist = 0; ierr = PetscHeaderDestroy(&C); CHKERRQ(ierr); PetscFunctionReturn(0); }
/*@ ISDestroy - Destroys an index set. Collective on IS Input Parameters: . is - the index set Level: beginner .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlocked() @*/ PetscErrorCode ISDestroy(IS *is) { PetscErrorCode ierr; PetscFunctionBegin; if (!*is) PetscFunctionReturn(0); PetscValidHeaderSpecific((*is),IS_CLASSID,1); if (--((PetscObject)(*is))->refct > 0) { *is = 0; PetscFunctionReturn(0); } if ((*is)->complement) { PetscInt refcnt; ierr = PetscObjectGetReference((PetscObject)((*is)->complement), &refcnt); CHKERRQ(ierr); if (refcnt > 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Nonlocal IS has not been restored"); ierr = ISDestroy(&(*is)->complement); CHKERRQ(ierr); } if ((*is)->ops->destroy) { ierr = (*(*is)->ops->destroy)(*is); CHKERRQ(ierr); } ierr = PetscLayoutDestroy(&(*is)->map); CHKERRQ(ierr); /* Destroy local representations of offproc data. */ ierr = PetscFree((*is)->total); CHKERRQ(ierr); ierr = PetscFree((*is)->nonlocal); CHKERRQ(ierr); ierr = PetscHeaderDestroy(is); CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode DMSetUp_Composite(DM dm) { PetscErrorCode ierr; PetscInt nprev = 0; PetscMPIInt rank,size; DM_Composite *com = (DM_Composite*)dm->data; struct DMCompositeLink *next = com->next; PetscLayout map; PetscFunctionBegin; if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup"); ierr = PetscLayoutCreate(((PetscObject)dm)->comm,&map);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(map,com->n);CHKERRQ(ierr); ierr = PetscLayoutSetSize(map,PETSC_DETERMINE);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(map);CHKERRQ(ierr); ierr = PetscLayoutGetSize(map,&com->N);CHKERRQ(ierr); ierr = PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr); /* now set the rstart for each linked vector */ ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(((PetscObject)dm)->comm,&size);CHKERRQ(ierr); while (next) { next->rstart = nprev; nprev += next->n; next->grstart = com->rstart + next->rstart; ierr = PetscMalloc(size*sizeof(PetscInt),&next->grstarts);CHKERRQ(ierr); ierr = MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);CHKERRQ(ierr); next = next->next; } com->setup = PETSC_TRUE; PetscFunctionReturn(0); }
/*@C PetscLayoutReference - Causes a PETSc Vec or Mat to share a PetscLayout with one that already exists. Used by Vec/MatDuplicate_XXX() Collective on PetscLayout Input Parameter: . in - input PetscLayout to be copied Output Parameter: . out - the reference location Level: developer Notes: PetscLayoutSetUp() does not need to be called on the resulting PetscLayout If the out location already contains a PetscLayout it is destroyed .seealso: PetscLayoutCreate(), PetscLayoutDestroy(), PetscLayoutSetUp(), PetscLayoutDuplicate() @*/ PetscErrorCode PetscLayoutReference(PetscLayout in,PetscLayout *out) { PetscErrorCode ierr; PetscFunctionBegin; in->refcnt++; ierr = PetscLayoutDestroy(out);CHKERRQ(ierr); *out = in; PetscFunctionReturn(0); }
PetscErrorCode AODestroy_MemoryScalable(AO ao) { AO_MemoryScalable *aomems = (AO_MemoryScalable*)ao->data; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscFree2(aomems->app_loc,aomems->petsc_loc);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&aomems->map);CHKERRQ(ierr); ierr = PetscFree(aomems);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode MatHeaderReplace(Mat A,Mat C) { PetscErrorCode ierr; PetscFunctionBegin; if (A == C) PetscFunctionReturn(0); /* free all the interior data structures from mat */ ierr = (*A->ops->destroy)(A);CHKERRQ(ierr); ierr = PetscHeaderDestroy_Private((PetscObject)A);CHKERRQ(ierr); ierr = PetscFree(A->ops);CHKERRQ(ierr); ierr = PetscLayoutDestroy(A->rmap);CHKERRQ(ierr); ierr = PetscLayoutDestroy(A->cmap);CHKERRQ(ierr); ierr = PetscFree(A->spptr);CHKERRQ(ierr); /* copy C over to A */ if (C) { ierr = PetscMemcpy(A,C,sizeof(struct _p_Mat));CHKERRQ(ierr); ierr = PetscLogObjectDestroy((PetscObject)C);CHKERRQ(ierr); ierr = PetscFree(C);CHKERRQ(ierr); } PetscFunctionReturn(0); }
/*@C PetscLayoutCopy - creates a new PetscLayout with the same information as a given one. If the PetscLayout already exists it is destroyed first. Collective on PetscLayout Input Parameter: . in - input PetscLayout to be copied Output Parameter: . out - the copy Level: developer Notes: PetscLayoutSetUp() does not need to be called on the resulting PetscLayout Developer Note: Unlike all other copy routines this destroys any input object and makes a new one. This routine should be fixed to have a PetscLayoutDuplicate() that ONLY creates a new one and a PetscLayoutCopy() that truely copies the data and does not delete the old object. .seealso: PetscLayoutCreate(), PetscLayoutDestroy(), PetscLayoutSetUp() @*/ PetscErrorCode PETSCVEC_DLLEXPORT PetscLayoutCopy(PetscLayout in,PetscLayout *out) { PetscMPIInt size; PetscErrorCode ierr; MPI_Comm comm = in->comm; PetscFunctionBegin; if (*out) {ierr = PetscLayoutDestroy(*out);CHKERRQ(ierr);} ierr = PetscLayoutCreate(comm,out);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = PetscMemcpy(*out,in,sizeof(struct _p_PetscLayout));CHKERRQ(ierr); ierr = PetscMalloc((size+1)*sizeof(PetscInt),&(*out)->range);CHKERRQ(ierr); ierr = PetscMemcpy((*out)->range,in->range,(size+1)*sizeof(PetscInt));CHKERRQ(ierr); (*out)->refcnt = 0; PetscFunctionReturn(0); }
/*@ PetscLayoutDuplicate - creates a new PetscLayout with the same information as a given one. If the PetscLayout already exists it is destroyed first. Collective on PetscLayout Input Parameter: . in - input PetscLayout to be duplicated Output Parameter: . out - the copy Level: developer Notes: PetscLayoutSetUp() does not need to be called on the resulting PetscLayout .seealso: PetscLayoutCreate(), PetscLayoutDestroy(), PetscLayoutSetUp(), PetscLayoutReference() @*/ PetscErrorCode PetscLayoutDuplicate(PetscLayout in,PetscLayout *out) { PetscMPIInt size; PetscErrorCode ierr; MPI_Comm comm = in->comm; PetscFunctionBegin; ierr = PetscLayoutDestroy(out);CHKERRQ(ierr); ierr = PetscLayoutCreate(comm,out);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = PetscMemcpy(*out,in,sizeof(struct _n_PetscLayout));CHKERRQ(ierr); ierr = PetscMalloc1(size+1,&(*out)->range);CHKERRQ(ierr); ierr = PetscMemcpy((*out)->range,in->range,(size+1)*sizeof(PetscInt));CHKERRQ(ierr); (*out)->refcnt = 0; PetscFunctionReturn(0); }
PetscErrorCode MatDestroy_MPIAIJ_PtAP(Mat A) { PetscErrorCode ierr; Mat_MPIAIJ *a =(Mat_MPIAIJ*)A->data; Mat_PtAPMPI *ptap=a->ptap; PetscFunctionBegin; if (ptap) { Mat_Merge_SeqsToMPI *merge=ptap->merge; ierr = PetscFree2(ptap->startsj_s,ptap->startsj_r);CHKERRQ(ierr); ierr = PetscFree(ptap->bufa);CHKERRQ(ierr); ierr = MatDestroy(&ptap->P_loc);CHKERRQ(ierr); ierr = MatDestroy(&ptap->P_oth);CHKERRQ(ierr); ierr = MatDestroy(&ptap->A_loc);CHKERRQ(ierr); /* used by MatTransposeMatMult() */ if (ptap->api) {ierr = PetscFree(ptap->api);CHKERRQ(ierr);} if (ptap->apj) {ierr = PetscFree(ptap->apj);CHKERRQ(ierr);} if (ptap->apa) {ierr = PetscFree(ptap->apa);CHKERRQ(ierr);} if (merge) { ierr = PetscFree(merge->id_r);CHKERRQ(ierr); ierr = PetscFree(merge->len_s);CHKERRQ(ierr); ierr = PetscFree(merge->len_r);CHKERRQ(ierr); ierr = PetscFree(merge->bi);CHKERRQ(ierr); ierr = PetscFree(merge->bj);CHKERRQ(ierr); ierr = PetscFree(merge->buf_ri[0]);CHKERRQ(ierr); ierr = PetscFree(merge->buf_ri);CHKERRQ(ierr); ierr = PetscFree(merge->buf_rj[0]);CHKERRQ(ierr); ierr = PetscFree(merge->buf_rj);CHKERRQ(ierr); ierr = PetscFree(merge->coi);CHKERRQ(ierr); ierr = PetscFree(merge->coj);CHKERRQ(ierr); ierr = PetscFree(merge->owners_co);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&merge->rowmap);CHKERRQ(ierr); ierr = merge->destroy(A);CHKERRQ(ierr); ierr = PetscFree(ptap->merge);CHKERRQ(ierr); } ierr = PetscFree(ptap);CHKERRQ(ierr); } PetscFunctionReturn(0); }
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer) { MPI_Comm comm; PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data; FILE *fp; PetscViewerVTKObjectLink link; PetscSection coordSection, globalCoordSection; PetscLayout vLayout; Vec coordinates; PetscReal lengthScale; PetscInt vMax, totVertices, totCells; PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr); ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr); /* Vertices */ ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr); ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr); ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr); ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr); ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr); if (vMax >= 0) { PetscInt pStart, pEnd, p, localSize = 0; ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr); pEnd = PetscMin(pEnd, vMax); for (p = pStart; p < pEnd; ++p) { PetscInt dof; ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr); if (dof > 0) ++localSize; } ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr); } else { ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr); } ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr); ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr); /* Cells */ ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr); /* Vertex fields */ for (link = vtk->link; link; link = link->next) { if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE; if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE; } if (hasPoint) { ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr); for (link = vtk->link; link; link = link->next) { Vec X = (Vec) link->vec; DM dmX; PetscSection section, globalSection, newSection = NULL; const char *name; PetscInt enforceDof = PETSC_DETERMINE; if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue; if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3; ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr); ierr = VecGetDM(X, &dmX);CHKERRQ(ierr); if (dmX) { DMLabel subpointMap, subpointMapX; PetscInt dim, dimX, pStart, pEnd, qStart, qEnd; ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr); /* Here is where we check whether dmX is a submesh of dm */ ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr); ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr); ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr); ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr); ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr); ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr); if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) { const PetscInt *ind = NULL; IS subpointIS; PetscInt n = 0, q; ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr); ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr); if (subpointIS) { ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr); ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr); } ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr); ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr); for (q = qStart; q < qEnd; ++q) { PetscInt dof, off, p; ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr); if (dof) { ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr); if (p >= pStart) { ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr); ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr); } } } if (subpointIS) { ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr); ierr = ISDestroy(&subpointIS);CHKERRQ(ierr); } /* No need to setup section */ section = newSection; } } else { ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr); if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); } if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr); ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr); ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr); if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);} } } /* Cell Fields */ ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr); if (hasCell || writePartition) { ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr); for (link = vtk->link; link; link = link->next) { Vec X = (Vec) link->vec; DM dmX; PetscSection section, globalSection; const char *name; PetscInt enforceDof = PETSC_DETERMINE; if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue; if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3; ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr); ierr = VecGetDM(X, &dmX);CHKERRQ(ierr); if (dmX) { ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr); } else { PetscContainer c; ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr); if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr); } if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr); ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr); ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr); } if (writePartition) { ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr); ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr); } } /* Cleanup */ ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr); ierr = PetscFClose(comm, fp);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix) { MPI_Comm comm; MatType mtype; PetscSF sf, sfDof, sfAdj; PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj; PetscInt nroots, nleaves, l, p; const PetscInt *leaves; const PetscSFNode *remotes; PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols; PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets; PetscInt adjSize; PetscLayout rLayout; PetscInt locRows, rStart, rEnd, r; PetscMPIInt size; PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock; PetscBool useAnchors; PetscErrorCode ierr; PetscFunctionBegin; PetscValidHeaderSpecific(dm, DM_CLASSID, 1); PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3); PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4); PetscValidHeaderSpecific(A, MAT_CLASSID, 9); if (dnz) PetscValidPointer(dnz,5); if (onz) PetscValidPointer(onz,6); if (dnzu) PetscValidPointer(dnzu,7); if (onzu) PetscValidPointer(onzu,8); ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr); ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr); ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr); ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr); ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr); ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr); ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr); doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE; ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr); /* Create dof SF based on point SF */ if (debug) { ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr); ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr); ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr); ierr = PetscSFView(sf, NULL);CHKERRQ(ierr); } ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr); ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr); if (debug) { ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr); ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr); } /* Create section for dof adjacency (dof ==> # adj dof) */ ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr); ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr); ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr); ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr); ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr); ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr); /* Fill in the ghost dofs on the interface */ ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr); /* use constraints in finding adjacency in this routine */ ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr); ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr); /* section - maps points to (# dofs, local dofs) sectionGlobal - maps points to (# dofs, global dofs) leafSectionAdj - maps unowned local dofs to # adj dofs rootSectionAdj - maps owned local dofs to # adj dofs adj - adj global dofs indexed by leafSectionAdj rootAdj - adj global dofs indexed by rootSectionAdj sf - describes shared points across procs sfDof - describes shared dofs across procs sfAdj - describes shared adjacent dofs across procs ** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point. (0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj (This is done in DMPlexComputeAnchorAdjacencies()) 1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj Reduce those counts to rootSectionAdj (now redundantly counting some interface points) 2. Visit owned points on interface, count adjacencies placing in rootSectionAdj Create sfAdj connecting rootSectionAdj and leafSectionAdj 3. Visit unowned points on interface, write adjacencies to adj Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies) 4. Visit owned points on interface, write adjacencies to rootAdj Remove redundancy in rootAdj ** The last two traversals use transitive closure 5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj) Allocate memory addressed by sectionAdj (cols) 6. Visit all owned points in the subdomain, insert dof adjacencies into cols ** Knowing all the column adjacencies, check ownership and sum into dnz and onz */ ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr); for (l = 0; l < nleaves; ++l) { PetscInt dof, off, d, q, anDof; PetscInt p = leaves[l], numAdj = PETSC_DETERMINE; if ((p < pStart) || (p >= pEnd)) continue; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr); for (q = 0; q < numAdj; ++q) { const PetscInt padj = tmpAdj[q]; PetscInt ndof, ncdof; if ((padj < pStart) || (padj >= pEnd)) continue; ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr); for (d = off; d < off+dof; ++d) { ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr); } } ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr); if (anDof) { for (d = off; d < off+dof; ++d) { ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr); } } } ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr); if (debug) { ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr); ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Get maximum remote adjacency sizes for owned dofs on interface (roots) */ if (doComm) { ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr); ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr); } if (debug) { ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr); ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Add in local adjacency sizes for owned dofs on interface (roots) */ for (p = pStart; p < pEnd; ++p) { PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); if (!dof) continue; ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr); if (adof <= 0) continue; ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr); for (q = 0; q < numAdj; ++q) { const PetscInt padj = tmpAdj[q]; PetscInt ndof, ncdof; if ((padj < pStart) || (padj >= pEnd)) continue; ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr); for (d = off; d < off+dof; ++d) { ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr); } } ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr); if (anDof) { for (d = off; d < off+dof; ++d) { ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr); } } } ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr); if (debug) { ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr); ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Create adj SF based on dof SF */ ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr); ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr); if (debug) { ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr); ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr); } ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr); /* Create leaf adjacency */ ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr); ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr); ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr); for (l = 0; l < nleaves; ++l) { PetscInt dof, off, d, q, anDof, anOff; PetscInt p = leaves[l], numAdj = PETSC_DETERMINE; if ((p < pStart) || (p >= pEnd)) continue; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr); ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr); for (d = off; d < off+dof; ++d) { PetscInt aoff, i = 0; ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr); for (q = 0; q < numAdj; ++q) { const PetscInt padj = tmpAdj[q]; PetscInt ndof, ncdof, ngoff, nd; if ((padj < pStart) || (padj >= pEnd)) continue; ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr); for (nd = 0; nd < ndof-ncdof; ++nd) { adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd; ++i; } } for (q = 0; q < anDof; q++) { adj[aoff+i] = anchorAdj[anOff+q]; ++i; } } } /* Debugging */ if (debug) { IS tmp; ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr); ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr); ierr = ISView(tmp, NULL);CHKERRQ(ierr); ierr = ISDestroy(&tmp);CHKERRQ(ierr); } /* Gather adjacenct indices to root */ ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr); ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr); for (r = 0; r < adjSize; ++r) rootAdj[r] = -1; if (doComm) { ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr); ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr); } ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr); ierr = PetscFree(adj);CHKERRQ(ierr); /* Debugging */ if (debug) { IS tmp; ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr); ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr); ierr = ISView(tmp, NULL);CHKERRQ(ierr); ierr = ISDestroy(&tmp);CHKERRQ(ierr); } /* Add in local adjacency indices for owned dofs on interface (roots) */ for (p = pStart; p < pEnd; ++p) { PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); if (!dof) continue; ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr); if (adof <= 0) continue; ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr); ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr); for (d = off; d < off+dof; ++d) { PetscInt adof, aoff, i; ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr); i = adof-1; for (q = 0; q < anDof; q++) { rootAdj[aoff+i] = anchorAdj[anOff+q]; --i; } for (q = 0; q < numAdj; ++q) { const PetscInt padj = tmpAdj[q]; PetscInt ndof, ncdof, ngoff, nd; if ((padj < pStart) || (padj >= pEnd)) continue; ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr); for (nd = 0; nd < ndof-ncdof; ++nd) { rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd; --i; } } } } /* Debugging */ if (debug) { IS tmp; ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr); ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr); ierr = ISView(tmp, NULL);CHKERRQ(ierr); ierr = ISDestroy(&tmp);CHKERRQ(ierr); } /* Compress indices */ ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr); for (p = pStart; p < pEnd; ++p) { PetscInt dof, cdof, off, d; PetscInt adof, aoff; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); if (!dof) continue; ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr); if (adof <= 0) continue; for (d = off; d < off+dof-cdof; ++d) { ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr); ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr); ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr); } } /* Debugging */ if (debug) { IS tmp; ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr); ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr); ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr); ierr = ISView(tmp, NULL);CHKERRQ(ierr); ierr = ISDestroy(&tmp);CHKERRQ(ierr); } /* Build adjacency section: Maps global indices to sets of adjacent global indices */ ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr); ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr); ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr); for (p = pStart; p < pEnd; ++p) { PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof; PetscBool found = PETSC_TRUE; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr); for (d = 0; d < dof-cdof; ++d) { PetscInt ldof, rdof; ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr); ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr); if (ldof > 0) { /* We do not own this point */ } else if (rdof > 0) { ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr); } else { found = PETSC_FALSE; } } if (found) continue; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr); ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr); for (q = 0; q < numAdj; ++q) { const PetscInt padj = tmpAdj[q]; PetscInt ndof, ncdof, noff; if ((padj < pStart) || (padj >= pEnd)) continue; ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr); for (d = goff; d < goff+dof-cdof; ++d) { ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr); } } ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr); if (anDof) { for (d = goff; d < goff+dof-cdof; ++d) { ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr); } } } ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr); if (debug) { ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr); ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Get adjacent indices */ ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr); ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr); for (p = pStart; p < pEnd; ++p) { PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff; PetscBool found = PETSC_TRUE; ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr); for (d = 0; d < dof-cdof; ++d) { PetscInt ldof, rdof; ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr); ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr); if (ldof > 0) { /* We do not own this point */ } else if (rdof > 0) { PetscInt aoff, roff; ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr); ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr); ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr); } else { found = PETSC_FALSE; } } if (found) continue; ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr); ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr); for (d = goff; d < goff+dof-cdof; ++d) { PetscInt adof, aoff, i = 0; ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr); for (q = 0; q < numAdj; ++q) { const PetscInt padj = tmpAdj[q]; PetscInt ndof, ncdof, ngoff, nd; const PetscInt *ncind; /* Adjacent points may not be in the section chart */ if ((padj < pStart) || (padj >= pEnd)) continue; ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr); ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr); for (nd = 0; nd < ndof-ncdof; ++nd, ++i) { cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd; } } for (q = 0; q < anDof; q++, i++) { cols[aoff+i] = anchorAdj[anOff + q]; } if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p); } } ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr); ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr); ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr); ierr = PetscFree(anchorAdj);CHKERRQ(ierr); ierr = PetscFree(rootAdj);CHKERRQ(ierr); ierr = PetscFree(tmpAdj);CHKERRQ(ierr); /* Debugging */ if (debug) { IS tmp; ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr); ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr); ierr = ISView(tmp, NULL);CHKERRQ(ierr); ierr = ISDestroy(&tmp);CHKERRQ(ierr); } /* Create allocation vectors from adjacency graph */ ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr); ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr); ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr); /* Only loop over blocks of rows */ if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs); for (r = rStart/bs; r < rEnd/bs; ++r) { const PetscInt row = r*bs; PetscInt numCols, cStart, c; ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr); for (c = cStart; c < cStart+numCols; ++c) { if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) { ++dnz[r-rStart]; if (cols[c] >= row) ++dnzu[r-rStart]; } else { ++onz[r-rStart]; if (cols[c] >= row) ++onzu[r-rStart]; } } } if (bs > 1) { for (r = 0; r < locRows/bs; ++r) { dnz[r] /= bs; onz[r] /= bs; dnzu[r] /= bs; onzu[r] /= bs; } } /* Set matrix pattern */ ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr); ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr); /* Check for symmetric storage */ ierr = MatGetType(A, &mtype);CHKERRQ(ierr); ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr); ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr); ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr); if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);} /* Fill matrix with zeros */ if (fillMatrix) { PetscScalar *values; PetscInt maxRowLen = 0; for (r = rStart; r < rEnd; ++r) { PetscInt len; ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr); maxRowLen = PetscMax(maxRowLen, len); } ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr); for (r = rStart; r < rEnd; ++r) { PetscInt numCols, cStart; ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr); ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr); ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr); } ierr = PetscFree(values);CHKERRQ(ierr); ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); } /* restore original useAnchors */ ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr); ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr); ierr = PetscFree(cols);CHKERRQ(ierr); ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr); PetscFunctionReturn(0); }
static PetscErrorCode PCSetUp_Redistribute(PC pc) { PC_Redistribute *red = (PC_Redistribute*)pc->data; PetscErrorCode ierr; MPI_Comm comm; PetscInt rstart,rend,i,nz,cnt,*rows,ncnt,dcnt,*drows; PetscLayout map,nmap; PetscMPIInt size,imdex,tag,n; PetscInt *source = PETSC_NULL; PetscMPIInt *nprocs = PETSC_NULL,nrecvs; PetscInt j,nsends; PetscInt *owner = PETSC_NULL,*starts = PETSC_NULL,count,slen; PetscInt *rvalues,*svalues,recvtotal; PetscMPIInt *onodes1,*olengths1; MPI_Request *send_waits = PETSC_NULL,*recv_waits = PETSC_NULL; MPI_Status recv_status,*send_status; Vec tvec,diag; Mat tmat; const PetscScalar *d; PetscFunctionBegin; if (pc->setupcalled) { ierr = KSPGetOperators(red->ksp,PETSC_NULL,&tmat,PETSC_NULL);CHKERRQ(ierr); ierr = MatGetSubMatrix(pc->pmat,red->is,red->is,MAT_REUSE_MATRIX,&tmat);CHKERRQ(ierr); ierr = KSPSetOperators(red->ksp,tmat,tmat,SAME_NONZERO_PATTERN);CHKERRQ(ierr); } else { PetscInt NN; ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = PetscObjectGetNewTag((PetscObject)pc,&tag);CHKERRQ(ierr); /* count non-diagonal rows on process */ ierr = MatGetOwnershipRange(pc->mat,&rstart,&rend);CHKERRQ(ierr); cnt = 0; for (i=rstart; i<rend; i++) { ierr = MatGetRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr); if (nz > 1) cnt++; ierr = MatRestoreRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr); } ierr = PetscMalloc(cnt*sizeof(PetscInt),&rows);CHKERRQ(ierr); ierr = PetscMalloc((rend - rstart - cnt)*sizeof(PetscInt),&drows);CHKERRQ(ierr); /* list non-diagonal rows on process */ cnt = 0; dcnt = 0; for (i=rstart; i<rend; i++) { ierr = MatGetRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr); if (nz > 1) rows[cnt++] = i; else drows[dcnt++] = i - rstart; ierr = MatRestoreRow(pc->mat,i,&nz,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr); } /* create PetscLayout for non-diagonal rows on each process */ ierr = PetscLayoutCreate(comm,&map);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(map,cnt);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(map);CHKERRQ(ierr); rstart = map->rstart; rend = map->rend; /* create PetscLayout for load-balanced non-diagonal rows on each process */ ierr = PetscLayoutCreate(comm,&nmap);CHKERRQ(ierr); ierr = MPI_Allreduce(&cnt,&ncnt,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr); ierr = PetscLayoutSetSize(nmap,ncnt);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(nmap,1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(nmap);CHKERRQ(ierr); ierr = MatGetSize(pc->pmat,&NN,PETSC_NULL);CHKERRQ(ierr); ierr = PetscInfo2(pc,"Number of diagonal rows eliminated %d, percentage eliminated %g\n",NN-ncnt,((PetscReal)(NN-ncnt))/((PetscReal)(NN)));CHKERRQ(ierr); /* this code is taken from VecScatterCreate_PtoS() Determines what rows need to be moved where to load balance the non-diagonal rows */ /* count number of contributors to each processor */ ierr = PetscMalloc2(size,PetscMPIInt,&nprocs,cnt,PetscInt,&owner);CHKERRQ(ierr); ierr = PetscMemzero(nprocs,size*sizeof(PetscMPIInt));CHKERRQ(ierr); j = 0; nsends = 0; for (i=rstart; i<rend; i++) { if (i < nmap->range[j]) j = 0; for (; j<size; j++) { if (i < nmap->range[j+1]) { if (!nprocs[j]++) nsends++; owner[i-rstart] = j; break; } } } /* inform other processors of number of messages and max length*/ ierr = PetscGatherNumberOfMessages(comm,PETSC_NULL,nprocs,&nrecvs);CHKERRQ(ierr); ierr = PetscGatherMessageLengths(comm,nsends,nrecvs,nprocs,&onodes1,&olengths1);CHKERRQ(ierr); ierr = PetscSortMPIIntWithArray(nrecvs,onodes1,olengths1);CHKERRQ(ierr); recvtotal = 0; for (i=0; i<nrecvs; i++) recvtotal += olengths1[i]; /* post receives: rvalues - rows I will own; count - nu */ ierr = PetscMalloc3(recvtotal,PetscInt,&rvalues,nrecvs,PetscInt,&source,nrecvs,MPI_Request,&recv_waits);CHKERRQ(ierr); count = 0; for (i=0; i<nrecvs; i++) { ierr = MPI_Irecv((rvalues+count),olengths1[i],MPIU_INT,onodes1[i],tag,comm,recv_waits+i);CHKERRQ(ierr); count += olengths1[i]; } /* do sends: 1) starts[i] gives the starting index in svalues for stuff going to the ith processor */ ierr = PetscMalloc3(cnt,PetscInt,&svalues,nsends,MPI_Request,&send_waits,size,PetscInt,&starts);CHKERRQ(ierr); starts[0] = 0; for (i=1; i<size; i++) { starts[i] = starts[i-1] + nprocs[i-1];} for (i=0; i<cnt; i++) { svalues[starts[owner[i]]++] = rows[i]; } for (i=0; i<cnt; i++) rows[i] = rows[i] - rstart; red->drows = drows; red->dcnt = dcnt; ierr = PetscFree(rows);CHKERRQ(ierr); starts[0] = 0; for (i=1; i<size; i++) { starts[i] = starts[i-1] + nprocs[i-1];} count = 0; for (i=0; i<size; i++) { if (nprocs[i]) { ierr = MPI_Isend(svalues+starts[i],nprocs[i],MPIU_INT,i,tag,comm,send_waits+count++);CHKERRQ(ierr); } } /* wait on receives */ count = nrecvs; slen = 0; while (count) { ierr = MPI_Waitany(nrecvs,recv_waits,&imdex,&recv_status);CHKERRQ(ierr); /* unpack receives into our local space */ ierr = MPI_Get_count(&recv_status,MPIU_INT,&n);CHKERRQ(ierr); slen += n; count--; } if (slen != recvtotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Total message lengths %D not expected %D",slen,recvtotal); ierr = ISCreateGeneral(comm,slen,rvalues,PETSC_COPY_VALUES,&red->is);CHKERRQ(ierr); /* free up all work space */ ierr = PetscFree(olengths1);CHKERRQ(ierr); ierr = PetscFree(onodes1);CHKERRQ(ierr); ierr = PetscFree3(rvalues,source,recv_waits);CHKERRQ(ierr); ierr = PetscFree2(nprocs,owner);CHKERRQ(ierr); if (nsends) { /* wait on sends */ ierr = PetscMalloc(nsends*sizeof(MPI_Status),&send_status);CHKERRQ(ierr); ierr = MPI_Waitall(nsends,send_waits,send_status);CHKERRQ(ierr); ierr = PetscFree(send_status);CHKERRQ(ierr); } ierr = PetscFree3(svalues,send_waits,starts);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&nmap);CHKERRQ(ierr); ierr = VecCreateMPI(comm,slen,PETSC_DETERMINE,&red->b);CHKERRQ(ierr); ierr = VecDuplicate(red->b,&red->x);CHKERRQ(ierr); ierr = MatGetVecs(pc->pmat,&tvec,PETSC_NULL);CHKERRQ(ierr); ierr = VecScatterCreate(tvec,red->is,red->b,PETSC_NULL,&red->scatter);CHKERRQ(ierr); ierr = VecDestroy(&tvec);CHKERRQ(ierr); ierr = MatGetSubMatrix(pc->pmat,red->is,red->is,MAT_INITIAL_MATRIX,&tmat);CHKERRQ(ierr); ierr = KSPSetOperators(red->ksp,tmat,tmat,SAME_NONZERO_PATTERN);CHKERRQ(ierr); ierr = MatDestroy(&tmat);CHKERRQ(ierr); } /* get diagonal portion of matrix */ ierr = PetscMalloc(red->dcnt*sizeof(PetscScalar),&red->diag);CHKERRQ(ierr); ierr = MatGetVecs(pc->pmat,&diag,PETSC_NULL);CHKERRQ(ierr); ierr = MatGetDiagonal(pc->pmat,diag);CHKERRQ(ierr); ierr = VecGetArrayRead(diag,&d);CHKERRQ(ierr); for (i=0; i<red->dcnt; i++) { red->diag[i] = 1.0/d[red->drows[i]]; } ierr = VecRestoreArrayRead(diag,&d);CHKERRQ(ierr); ierr = VecDestroy(&diag);CHKERRQ(ierr); ierr = KSPSetUp(red->ksp);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints) { MPI_Comm comm = ctx->comm; PetscScalar *a; PetscInt p, q, i; PetscMPIInt rank, size; PetscErrorCode ierr; Vec pointVec; IS cellIS; PetscLayout layout; PetscReal *globalPoints; PetscScalar *globalPointsScalar; const PetscInt *ranges; PetscMPIInt *counts, *displs; const PetscInt *foundCells; PetscMPIInt *foundProcs, *globalProcs; PetscInt n, N; PetscFunctionBegin; PetscValidHeaderSpecific(dm, DM_CLASSID, 1); ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr); if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set"); /* Locate points */ n = ctx->nInput; if (!redundantPoints) { ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr); ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr); ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr); /* Communicate all points to all processes */ ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr); ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr); for (p = 0; p < size; ++p) { counts[p] = (ranges[p+1] - ranges[p])*ctx->dim; displs[p] = ranges[p]*ctx->dim; } ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr); } else { N = n; globalPoints = ctx->points; counts = displs = NULL; layout = NULL; } #if 0 ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr); /* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */ #else #if defined(PETSC_USE_COMPLEX) ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr); for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i]; #else globalPointsScalar = globalPoints; #endif ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr); ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr); ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr); ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr); #endif for (p = 0; p < N; ++p) { if (foundCells[p] >= 0) foundProcs[p] = rank; else foundProcs[p] = size; } /* Let the lowest rank process own each point */ ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr); ctx->n = 0; for (p = 0; p < N; ++p) { if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0); else if (globalProcs[p] == rank) ctx->n++; } /* Create coordinates vector and array of owned cells */ ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr); ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr); ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr); ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr); ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr); for (p = 0, q = 0, i = 0; p < N; ++p) { if (globalProcs[p] == rank) { PetscInt d; for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d]; ctx->cells[q++] = foundCells[p]; } } ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr); #if 0 ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr); #else ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr); ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr); ierr = ISDestroy(&cellIS);CHKERRQ(ierr); ierr = VecDestroy(&pointVec);CHKERRQ(ierr); #endif if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);} if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);} ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode MatLoad_AIJ_HDF5(Mat mat, PetscViewer viewer) { PetscViewerFormat format; const PetscInt *i_glob = NULL; PetscInt *i = NULL; const PetscInt *j = NULL; const PetscScalar *a = NULL; const char *a_name = NULL, *i_name = NULL, *j_name = NULL, *mat_name = NULL, *c_name = NULL; PetscInt p, m, M, N; PetscInt bs = mat->rmap->bs; PetscBool flg; IS is_i = NULL, is_j = NULL; Vec vec_a = NULL; PetscLayout jmap = NULL; MPI_Comm comm; PetscMPIInt rank, size; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr); switch (format) { case PETSC_VIEWER_HDF5_PETSC: case PETSC_VIEWER_DEFAULT: case PETSC_VIEWER_NATIVE: case PETSC_VIEWER_HDF5_MAT: break; default: SETERRQ1(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"PetscViewerFormat %s not supported for HDF5 input.",PetscViewerFormats[format]); } ierr = PetscObjectGetComm((PetscObject)mat,&comm);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = PetscObjectGetName((PetscObject)mat,&mat_name);CHKERRQ(ierr); ierr = PetscViewerHDF5GetAIJNames(viewer,&i_name,&j_name,&a_name,&c_name);CHKERRQ(ierr); ierr = PetscOptionsBegin(comm,NULL,"Options for loading matrix from HDF5","Mat");CHKERRQ(ierr); ierr = PetscOptionsInt("-matload_block_size","Set the blocksize used to store the matrix","MatLoad",bs,&bs,&flg);CHKERRQ(ierr); ierr = PetscOptionsEnd();CHKERRQ(ierr); if (flg) { ierr = MatSetBlockSize(mat, bs);CHKERRQ(ierr); } ierr = PetscViewerHDF5PushGroup(viewer,mat_name);CHKERRQ(ierr); ierr = PetscViewerHDF5ReadAttribute(viewer,NULL,c_name,PETSC_INT,&N);CHKERRQ(ierr); ierr = PetscViewerHDF5ReadSizes(viewer, i_name, NULL, &M);CHKERRQ(ierr); --M; /* i has size M+1 as there is global number of nonzeros stored at the end */ if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) { /* Swap row and columns layout for unallocated matrix. I want to avoid calling MatTranspose() just to transpose sparsity pattern and layout. */ if (!mat->preallocated) { PetscLayout tmp; tmp = mat->rmap; mat->rmap = mat->cmap; mat->cmap = tmp; } else SETERRQ(comm,PETSC_ERR_SUP,"Not for preallocated matrix - we would need to transpose it here which we want to avoid"); } /* If global sizes are set, check if they are consistent with that given in the file */ if (mat->rmap->N >= 0 && mat->rmap->N != M) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of rows: Matrix in file has (%D) and input matrix has (%D)",mat->rmap->N,M); if (mat->cmap->N >= 0 && mat->cmap->N != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of cols: Matrix in file has (%D) and input matrix has (%D)",mat->cmap->N,N); /* Determine ownership of all (block) rows and columns */ mat->rmap->N = M; mat->cmap->N = N; ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr); ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr); m = mat->rmap->n; /* Read array i (array of row indices) */ ierr = PetscMalloc1(m+1, &i);CHKERRQ(ierr); /* allocate i with one more position for local number of nonzeros on each rank */ i[0] = i[m] = 0; /* make the last entry always defined - the code block below overwrites it just on last rank */ if (rank == size-1) m++; /* in the loaded array i_glob, only the last rank has one more position with the global number of nonzeros */ M++; ierr = ISCreate(comm,&is_i);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject)is_i,i_name);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(is_i->map,m);CHKERRQ(ierr); ierr = PetscLayoutSetSize(is_i->map,M);CHKERRQ(ierr); ierr = ISLoad(is_i,viewer);CHKERRQ(ierr); ierr = ISGetIndices(is_i,&i_glob);CHKERRQ(ierr); ierr = PetscMemcpy(i,i_glob,m*sizeof(PetscInt));CHKERRQ(ierr); /* Reset m and M to the matrix sizes */ m = mat->rmap->n; M--; /* Create PetscLayout for j and a vectors; construct ranges first */ ierr = PetscLayoutCreate(comm,&jmap);CHKERRQ(ierr); ierr = PetscCalloc1(size+1, &jmap->range);CHKERRQ(ierr); ierr = MPI_Allgather(&i[0], 1, MPIU_INT, jmap->range, 1, MPIU_INT, comm);CHKERRQ(ierr); /* Last rank has global number of nonzeros (= length of j and a arrays) in i[m] (last i entry) so broadcast it */ jmap->range[size] = i[m]; ierr = MPI_Bcast(&jmap->range[size], 1, MPIU_INT, size-1, comm);CHKERRQ(ierr); for (p=size-1; p>0; p--) { if (!jmap->range[p]) jmap->range[p] = jmap->range[p+1]; /* for ranks with 0 rows, take the value from the next processor */ } i[m] = jmap->range[rank+1]; /* i[m] (last i entry) is equal to next rank's offset */ /* Deduce rstart, rend, n and N from the ranges */ ierr = PetscLayoutSetUp(jmap);CHKERRQ(ierr); /* Convert global to local indexing of rows */ for (p=1; p<m+1; ++p) i[p] -= i[0]; i[0] = 0; /* Read array j (array of column indices) */ ierr = ISCreate(comm,&is_j);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject)is_j,j_name);CHKERRQ(ierr); ierr = PetscLayoutDuplicate(jmap,&is_j->map);CHKERRQ(ierr); ierr = ISLoad(is_j,viewer);CHKERRQ(ierr); ierr = ISGetIndices(is_j,&j);CHKERRQ(ierr); /* Read array a (array of values) */ ierr = VecCreate(comm,&vec_a);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject)vec_a,a_name);CHKERRQ(ierr); ierr = PetscLayoutDuplicate(jmap,&vec_a->map);CHKERRQ(ierr); ierr = VecLoad(vec_a,viewer);CHKERRQ(ierr); ierr = VecGetArrayRead(vec_a,&a);CHKERRQ(ierr); /* populate matrix */ if (!((PetscObject)mat)->type_name) { ierr = MatSetType(mat,MATAIJ);CHKERRQ(ierr); } ierr = MatSeqAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr); ierr = MatMPIAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr); /* ierr = MatSeqBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr); ierr = MatMPIBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr); */ if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) { /* Transpose the input matrix back */ ierr = MatTranspose(mat,MAT_INPLACE_MATRIX,&mat);CHKERRQ(ierr); } ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&jmap);CHKERRQ(ierr); ierr = PetscFree(i);CHKERRQ(ierr); ierr = ISRestoreIndices(is_i,&i_glob);CHKERRQ(ierr); ierr = ISRestoreIndices(is_j,&j);CHKERRQ(ierr); ierr = VecRestoreArrayRead(vec_a,&a);CHKERRQ(ierr); ierr = ISDestroy(&is_i);CHKERRQ(ierr); ierr = ISDestroy(&is_j);CHKERRQ(ierr); ierr = VecDestroy(&vec_a);CHKERRQ(ierr); PetscFunctionReturn(0); }