PetscErrorCode VecLoad_Binary(Vec vec, PetscViewer viewer)
{
PetscMPIInt size,rank,tag;
int fd;
PetscInt i,rows = 0,n,*range,N,bs;
PetscErrorCode ierr;
PetscBool flag;
PetscScalar *avec,*avecwork;
MPI_Comm comm;
MPI_Request request;
MPI_Status status;
#if defined(PETSC_HAVE_MPIIO)
PetscBool useMPIIO;
#endif
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(viewer,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryReadVecHeader_Private(viewer,&rows);CHKERRQ(ierr);
/* Set Vec sizes,blocksize,and type if not already set. Block size first so that local sizes will be compatible. */
ierr = PetscOptionsGetInt(((PetscObject)vec)->prefix, "-vecload_block_size", &bs, &flag);CHKERRQ(ierr);
if (flag) {
ierr = VecSetBlockSize(vec, bs);CHKERRQ(ierr);
}
if (vec->map->n < 0 && vec->map->N < 0) {
ierr = VecSetSizes(vec,PETSC_DECIDE,rows);CHKERRQ(ierr);
}
/* If sizes and type already set,check if the vector global size is correct */
ierr = VecGetSize(vec, &N);CHKERRQ(ierr);
if (N != rows) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Vector in file different length (%d) then input vector (%d)", rows, N);
#if defined(PETSC_HAVE_MPIIO)
ierr = PetscViewerBinaryGetMPIIO(viewer,&useMPIIO);CHKERRQ(ierr);
if (useMPIIO) {
ierr = VecLoad_Binary_MPIIO(vec, viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
ierr = VecGetLocalSize(vec,&n);CHKERRQ(ierr);
ierr = PetscObjectGetNewTag((PetscObject)viewer,&tag);CHKERRQ(ierr);
ierr = VecGetArray(vec,&avec);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryRead(fd,avec,n,PETSC_SCALAR);CHKERRQ(ierr);
if (size > 1) {
/* read in other chuncks and send to other processors */
/* determine maximum chunck owned by other */
range = vec->map->range;
n = 1;
for (i=1; i<size; i++) n = PetscMax(n,range[i+1] - range[i]);
ierr = PetscMalloc1(n,&avecwork);CHKERRQ(ierr);
for (i=1; i<size; i++) {
n = range[i+1] - range[i];
ierr = PetscBinaryRead(fd,avecwork,n,PETSC_SCALAR);CHKERRQ(ierr);
ierr = MPI_Isend(avecwork,n,MPIU_SCALAR,i,tag,comm,&request);CHKERRQ(ierr);
ierr = MPI_Wait(&request,&status);CHKERRQ(ierr);
}
ierr = PetscFree(avecwork);CHKERRQ(ierr);
}
} else {
ierr = MPI_Recv(avec,n,MPIU_SCALAR,0,tag,comm,&status);CHKERRQ(ierr);
}
ierr = VecRestoreArray(vec,&avec);CHKERRQ(ierr);
ierr = VecAssemblyBegin(vec);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCreateExodus - Create a DMPlex mesh from an ExodusII file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. exoid - The ExodusII id associated with a exodus file and obtained using ex_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Level: beginner
.keywords: mesh,ExodusII
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateExodus(MPI_Comm comm, PetscInt exoid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_EXODUSII)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from ex_get_init() */
char title[PETSC_MAX_PATH_LEN+1];
int dim = 0, numVertices = 0, numCells = 0;
int num_cs = 0, num_vs = 0, num_fs = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_EXODUSII)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open EXODUS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
ierr = PetscMemzero(title,(PETSC_MAX_PATH_LEN+1)*sizeof(char));CHKERRQ(ierr);
ierr = ex_get_init(exoid, title, &dim, &numVertices, &numCells, &num_cs, &num_vs, &num_fs);
if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ExodusII ex_get_init() failed with error code %D\n",ierr);
if (!num_cs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Exodus file does not contain any cell set\n");
}
ierr = MPI_Bcast(title, PETSC_MAX_PATH_LEN+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, title);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read cell sets information */
if (!rank) {
PetscInt *cone;
int c, cs, c_loc, v, v_loc;
/* Read from ex_get_elem_blk_ids() */
int *cs_id;
/* Read from ex_get_elem_block() */
char buffer[PETSC_MAX_PATH_LEN+1];
int num_cell_in_set, num_vertex_per_cell, num_attr;
/* Read from ex_get_elem_conn() */
int *cs_connect;
/* Get cell sets IDs */
ierr = PetscMalloc1(num_cs, &cs_id);CHKERRQ(ierr);
ierr = ex_get_elem_blk_ids(exoid, cs_id);CHKERRQ(ierr);
/* Read the cell set connectivity table and build mesh topology
EXO standard requires that cells in cell sets be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
for (c_loc = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, num_vertex_per_cell);CHKERRQ(ierr);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc2(num_vertex_per_cell*num_cell_in_set,&cs_connect,num_vertex_per_cell,&cone);CHKERRQ(ierr);
ierr = ex_get_elem_conn(exoid, cs_id[cs], cs_connect);CHKERRQ(ierr);
/* EXO uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
for (v_loc = 0; v_loc < num_vertex_per_cell; ++v_loc, ++v) {
cone[v_loc] = cs_connect[v]+numCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (num_vertex_per_cell == 4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (num_vertex_per_cell == 8) {
PetscInt tmp = cone[1];
cone[1] = cone[3];
cone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMSetLabelValue(*dm, "Cell Sets", c, cs_id[cs]);CHKERRQ(ierr);
}
ierr = PetscFree2(cs_connect,cone);CHKERRQ(ierr);
}
ierr = PetscFree(cs_id);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain Cell Sets label */
{
DMLabel label;
ierr = DMRemoveLabel(*dm, "Cell Sets", &label);CHKERRQ(ierr);
if (label) {ierr = DMAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Create vertex set label */
if (!rank && (num_vs > 0)) {
int vs, v;
/* Read from ex_get_node_set_ids() */
int *vs_id;
/* Read from ex_get_node_set_param() */
int num_vertex_in_set, num_attr;
/* Read from ex_get_node_set() */
int *vs_vertex_list;
/* Get vertex set ids */
ierr = PetscMalloc1(num_vs, &vs_id);CHKERRQ(ierr);
ierr = ex_get_node_set_ids(exoid, vs_id);CHKERRQ(ierr);
for (vs = 0; vs < num_vs; ++vs) {
ierr = ex_get_node_set_param(exoid, vs_id[vs], &num_vertex_in_set, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc1(num_vertex_in_set, &vs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_node_set(exoid, vs_id[vs], vs_vertex_list);CHKERRQ(ierr);
for (v = 0; v < num_vertex_in_set; ++v) {
ierr = DMSetLabelValue(*dm, "Vertex Sets", vs_vertex_list[v]+numCells-1, vs_id[vs]);CHKERRQ(ierr);
}
ierr = PetscFree(vs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(vs_id);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
float *x, *y, *z;
ierr = PetscMalloc3(numVertices,&x,numVertices,&y,numVertices,&z);CHKERRQ(ierr);
ierr = ex_get_coord(exoid, x, y, z);CHKERRQ(ierr);
if (dim > 0) {
for (v = 0; v < numVertices; ++v) coords[v*dim+0] = x[v];
}
if (dim > 1) {
for (v = 0; v < numVertices; ++v) coords[v*dim+1] = y[v];
}
if (dim > 2) {
for (v = 0; v < numVertices; ++v) coords[v*dim+2] = z[v];
}
ierr = PetscFree3(x,y,z);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Create side set label */
if (!rank && interpolate && (num_fs > 0)) {
int fs, f, voff;
/* Read from ex_get_side_set_ids() */
int *fs_id;
/* Read from ex_get_side_set_param() */
int num_side_in_set, num_dist_fact_in_set;
/* Read from ex_get_side_set_node_list() */
int *fs_vertex_count_list, *fs_vertex_list;
/* Get side set ids */
ierr = PetscMalloc1(num_fs, &fs_id);CHKERRQ(ierr);
ierr = ex_get_side_set_ids(exoid, fs_id);CHKERRQ(ierr);
for (fs = 0; fs < num_fs; ++fs) {
ierr = ex_get_side_set_param(exoid, fs_id[fs], &num_side_in_set, &num_dist_fact_in_set);CHKERRQ(ierr);
ierr = PetscMalloc2(num_side_in_set,&fs_vertex_count_list,num_side_in_set*4,&fs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_side_set_node_list(exoid, fs_id[fs], fs_vertex_count_list, fs_vertex_list);CHKERRQ(ierr);
for (f = 0, voff = 0; f < num_side_in_set; ++f) {
const PetscInt *faces = NULL;
PetscInt faceSize = fs_vertex_count_list[f], numFaces;
PetscInt faceVertices[4], v;
if (faceSize > 4) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "ExodusII side cannot have %d > 4 vertices", faceSize);
for (v = 0; v < faceSize; ++v, ++voff) {
faceVertices[v] = fs_vertex_list[voff]+numCells-1;
}
ierr = DMPlexGetFullJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Invalid ExodusII side %d in set %d maps to %d faces", f, fs, numFaces);
ierr = DMSetLabelValue(*dm, "Face Sets", faces[0], fs_id[fs]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
}
ierr = PetscFree2(fs_vertex_count_list,fs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(fs_id);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires ExodusII support. Reconfigure using --download-exodusii");
#endif
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
Vec vec, tagged, untagged;
VecScatter taggedScatter, untaggedScatter;
PetscInt bs;
PetscInt n, nloc, nint, i, j, k, localStart, localEnd, ntagged, nuntagged;
MPI_Comm comm;
VecTagger tagger;
PetscScalar *array;
PetscRandom rand;
VecTaggerBox *defaultBox;
VecTaggerBox *boxes;
IS is, isBlockGlobal, isComp;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
n = 10.;
bs = 1;
comm = PETSC_COMM_WORLD;
ierr = PetscOptionsBegin(comm, "" , "VecTagger Test Options", "Vec");CHKERRQ(ierr);
ierr = PetscOptionsInt("-bs","The block size of the vector","ex1.c",bs,&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-n","The size of the vector (in blocks)","ex1.c",n,&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscRandomCreate(comm,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = VecCreate(comm,&vec);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)vec,"Vec to Tag");CHKERRQ(ierr);
ierr = VecSetBlockSize(vec,bs);CHKERRQ(ierr);
ierr = VecSetSizes(vec,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetUp(vec);CHKERRQ(ierr);
ierr = VecGetLocalSize(vec,&nloc);CHKERRQ(ierr);
ierr = VecGetArray(vec,&array);CHKERRQ(ierr);
for (i = 0; i < nloc; i++) {
PetscScalar val;
ierr = PetscRandomGetValue(rand,&val);CHKERRQ(ierr);
array[i] = val;
}
ierr = VecRestoreArray(vec,&array);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = VecViewFromOptions(vec,NULL,"-vec_view");CHKERRQ(ierr);
ierr = VecTaggerCreate(comm,&tagger);CHKERRQ(ierr);
ierr = VecTaggerSetBlockSize(tagger,bs);CHKERRQ(ierr);
ierr = VecTaggerSetType(tagger,VECTAGGERABSOLUTE);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&defaultBox);CHKERRQ(ierr);
for (i = 0; i < bs; i++) {
#if !defined(PETSC_USE_COMPLEX)
defaultBox[i].min = 0.1;
defaultBox[i].max = 1.5;
#else
defaultBox[i].min = PetscCMPLX(0.1,0.1);
defaultBox[i].max = PetscCMPLX(1.5,1.5);
#endif
}
ierr = VecTaggerAbsoluteSetBox(tagger,defaultBox);CHKERRQ(ierr);
ierr = PetscFree(defaultBox);CHKERRQ(ierr);
ierr = VecTaggerSetFromOptions(tagger);CHKERRQ(ierr);
ierr = VecTaggerSetUp(tagger);CHKERRQ(ierr);
ierr = PetscObjectViewFromOptions((PetscObject)tagger,NULL,"-vec_tagger_view");CHKERRQ(ierr);
ierr = VecTaggerGetBlockSize(tagger,&bs);CHKERRQ(ierr);
ierr = VecTaggerComputeBoxes(tagger,vec,&nint,&boxes);
if (ierr && ierr != PETSC_ERR_SUP) CHKERRQ(ierr);
else {
PetscViewer viewer = NULL;
ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-vec_tagger_boxes_view",&viewer,NULL,NULL);CHKERRQ(ierr);
if (viewer) {
PetscBool iascii;
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
if (iascii) {
ierr = PetscViewerASCIIPrintf(viewer,"Num boxes: %D\n",nint);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
for (i = 0, k = 0; i < nint; i++) {
ierr = PetscViewerASCIIPrintf(viewer,"%D: ",i);CHKERRQ(ierr);
for (j = 0; j < bs; j++, k++) {
if (j) {ierr = PetscViewerASCIIPrintf(viewer," x ");CHKERRQ(ierr);}
#if !defined(PETSC_USE_COMPLEX)
ierr = PetscViewerASCIIPrintf(viewer,"[%g,%g]",(double)boxes[k].min,(double)boxes[k].max);CHKERRQ(ierr);
#else
ierr = PetscViewerASCIIPrintf(viewer,"[%g+%gi,%g+%gi]",(double)PetscRealPart(boxes[k].min),(double)PetscImaginaryPart(boxes[k].min),(double)PetscRealPart(boxes[k].max),(double)PetscImaginaryPart(boxes[k].max));CHKERRQ(ierr);
#endif
}
ierr = PetscViewerASCIIPrintf(viewer,"\n");CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
}
}
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscFree(boxes);CHKERRQ(ierr);
}
ierr = VecTaggerComputeIS(tagger,vec,&is);CHKERRQ(ierr);
ierr = ISGetBlockGlobalIS(is,vec,bs,&isBlockGlobal);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)isBlockGlobal,"Tagged IS (block global)");CHKERRQ(ierr);
ierr = ISViewFromOptions(isBlockGlobal,NULL,"-tagged_is_view");CHKERRQ(ierr);
ierr = VecGetOwnershipRange(vec,&localStart,&localEnd);CHKERRQ(ierr);
ierr = ISComplement(isBlockGlobal,localStart,localEnd,&isComp);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)isComp,"Untagged IS (global)");CHKERRQ(ierr);
ierr = ISViewFromOptions(isComp,NULL,"-untagged_is_view");CHKERRQ(ierr);
ierr = ISGetLocalSize(isBlockGlobal,&ntagged);CHKERRQ(ierr);
ierr = ISGetLocalSize(isComp,&nuntagged);CHKERRQ(ierr);
ierr = VecCreate(comm,&tagged);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)tagged,"Tagged selection");CHKERRQ(ierr);
ierr = VecSetSizes(tagged,ntagged,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetUp(tagged);CHKERRQ(ierr);
ierr = VecCreate(comm,&untagged);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)untagged,"Untagged selection");CHKERRQ(ierr);
ierr = VecSetSizes(untagged,nuntagged,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetUp(untagged);CHKERRQ(ierr);
ierr = VecScatterCreate(vec,isBlockGlobal,tagged,NULL,&taggedScatter);CHKERRQ(ierr);
ierr = VecScatterCreate(vec,isComp,untagged,NULL,&untaggedScatter);CHKERRQ(ierr);
ierr = VecScatterBegin(taggedScatter,vec,tagged,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(taggedScatter,vec,tagged,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterBegin(untaggedScatter,vec,untagged,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(untaggedScatter,vec,untagged,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecViewFromOptions(tagged,NULL,"-tagged_vec_view");CHKERRQ(ierr);
ierr = VecViewFromOptions(untagged,NULL,"-untagged_vec_view");CHKERRQ(ierr);
ierr = VecScatterDestroy(&untaggedScatter);CHKERRQ(ierr);
ierr = VecScatterDestroy(&taggedScatter);CHKERRQ(ierr);
ierr = VecDestroy(&untagged);CHKERRQ(ierr);
ierr = VecDestroy(&tagged);CHKERRQ(ierr);
ierr = ISDestroy(&isComp);CHKERRQ(ierr);
ierr = ISDestroy(&isBlockGlobal);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecTaggerDestroy(&tagger);CHKERRQ(ierr);
ierr = VecDestroy(&vec);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
This should handle properly the cases where PetscInt is 32 or 64 and hsize_t is 32 or 64. These means properly casting with
checks back and forth between the two types of variables.
*/
PetscErrorCode VecLoad_HDF5(Vec xin, PetscViewer viewer)
{
hid_t file_id, group, dset_id, filespace, memspace, plist_id;
hsize_t rdim, dim;
hsize_t dims[4], count[4], offset[4];
herr_t status;
PetscInt n, N, bs = 1, bsInd, lenInd, low, timestep;
PetscScalar *x;
const char *vecname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerHDF5OpenGroup(viewer, &file_id, &group);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetTimestep(viewer, ×tep);CHKERRQ(ierr);
ierr = VecGetBlockSize(xin,&bs);CHKERRQ(ierr);
/* Create the dataset with default properties and close filespace */
ierr = PetscObjectGetName((PetscObject)xin,&vecname);CHKERRQ(ierr);
#if (H5_VERS_MAJOR * 10000 + H5_VERS_MINOR * 100 + H5_VERS_RELEASE >= 10800)
dset_id = H5Dopen2(group, vecname, H5P_DEFAULT);
#else
dset_id = H5Dopen(group, vecname);
#endif
if (dset_id == -1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Dopen() with Vec named %s",vecname);
/* Retrieve the dataspace for the dataset */
filespace = H5Dget_space(dset_id);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Dget_space()");
dim = 0;
if (timestep >= 0) ++dim;
++dim;
if (bs >= 1) ++dim;
#if defined(PETSC_USE_COMPLEX)
++dim;
#endif
rdim = H5Sget_simple_extent_dims(filespace, dims, NULL);
#if defined(PETSC_USE_COMPLEX)
bsInd = rdim-2;
#else
bsInd = rdim-1;
#endif
lenInd = timestep >= 0 ? 1 : 0;
if (rdim != dim) {
if (rdim == dim+1 && bs == -1) bs = dims[bsInd];
else SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Dimension of array in file %d not %d as expected",rdim,dim);
} else if (bs >= 1 && bs != (PetscInt) dims[bsInd]) {
ierr = VecSetBlockSize(xin, dims[bsInd]);CHKERRQ(ierr);
if (ierr) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Block size %d specified for vector does not match blocksize in file %d",bs,dims[bsInd]);
bs = dims[bsInd];
}
/* Set Vec sizes,blocksize,and type if not already set */
if ((xin)->map->n < 0 && (xin)->map->N < 0) {
ierr = VecSetSizes(xin, PETSC_DECIDE, dims[lenInd]*bs);CHKERRQ(ierr);
}
/* If sizes and type already set,check if the vector global size is correct */
ierr = VecGetSize(xin, &N);CHKERRQ(ierr);
if (N/bs != (PetscInt) dims[lenInd]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Vector in file different length (%d) then input vector (%d)", (PetscInt) dims[lenInd], N/bs);
/* Each process defines a dataset and reads it from the hyperslab in the file */
ierr = VecGetLocalSize(xin, &n);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
count[dim] = 1;
++dim;
}
ierr = PetscHDF5IntCast(n/bs,count + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
count[dim] = bs;
++dim;
}
#if defined(PETSC_USE_COMPLEX)
count[dim] = 2;
++dim;
#endif
memspace = H5Screate_simple(dim, count, NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Screate_simple()");
/* Select hyperslab in the file */
ierr = VecGetOwnershipRange(xin, &low, NULL);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
offset[dim] = timestep;
++dim;
}
ierr = PetscHDF5IntCast(low/bs,offset + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
offset[dim] = 0;
++dim;
}
#if defined(PETSC_USE_COMPLEX)
offset[dim] = 0;
++dim;
#endif
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);CHKERRQ(status);
/* Create property list for collective dataset read */
plist_id = H5Pcreate(H5P_DATASET_XFER);
if (plist_id == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Pcreate()");
#if defined(PETSC_HAVE_H5PSET_FAPL_MPIO)
status = H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);CHKERRQ(status);
#endif
/* To write dataset independently use H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT) */
ierr = VecGetArray(xin, &x);CHKERRQ(ierr);
status = H5Dread(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, x);CHKERRQ(status);
ierr = VecRestoreArray(xin, &x);CHKERRQ(ierr);
/* Close/release resources */
if (group != file_id) {
status = H5Gclose(group);CHKERRQ(status);
}
status = H5Pclose(plist_id);CHKERRQ(status);
status = H5Sclose(filespace);CHKERRQ(status);
status = H5Sclose(memspace);CHKERRQ(status);
status = H5Dclose(dset_id);CHKERRQ(status);
ierr = VecAssemblyBegin(xin);CHKERRQ(ierr);
ierr = VecAssemblyEnd(xin);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char** argv)
{
PC pc;
PetscErrorCode ierr;
PetscInt m, nn, M, j, k, ne = 4;
PetscReal* coords;
Vec x, rhs;
Mat A;
KSP ksp;
PetscMPIInt npe, rank;
PetscInitialize(&argc, &argv, NULL, NULL);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &npe);
CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "Linear elasticity in 3D", "");
{
char nestring[256];
ierr = PetscSNPrintf(nestring, sizeof nestring, "number of elements in each direction, ne+1 must be a multiple of %D (sizes^{1/3})",
(PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5));
ierr = PetscOptionsInt("-ne", nestring, "", ne, &ne, NULL);
}
ierr = PetscOptionsEnd();
CHKERRQ(ierr);
const HpddmOption* const opt = HpddmOptionGet();
{
HpddmOptionParse(opt, argc, argv, rank == 0);
if (rank) HpddmOptionRemove(opt, "verbosity");
}
nn = ne + 1;
M = 3 * nn * nn * nn;
if (npe == 2) {
if (rank == 1)
m = 0;
else
m = nn * nn * nn;
npe = 1;
}
else {
m = nn * nn * nn / npe;
if (rank == npe - 1) m = nn * nn * nn - (npe - 1) * m;
}
m *= 3;
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);
CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);
CHKERRQ(ierr);
int i;
{
PetscInt Istart, Iend, jj, ic;
const PetscInt NP = (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5);
const PetscInt ipx = rank % NP, ipy = (rank % (NP * NP)) / NP, ipz = rank / (NP * NP);
const PetscInt Ni0 = ipx * (nn / NP), Nj0 = ipy * (nn / NP), Nk0 = ipz * (nn / NP);
const PetscInt Ni1 = Ni0 + (m > 0 ? (nn / NP) : 0), Nj1 = Nj0 + (nn / NP), Nk1 = Nk0 + (nn / NP);
PetscInt *d_nnz, *o_nnz, osz[4] = {0, 9, 15, 19}, nbc;
if (npe != NP * NP * NP) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "npe=%d: npe^{1/3} must be integer", npe);
if (nn != NP * (nn / NP)) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "-ne %d: (ne+1)%(npe^{1/3}) must equal zero", ne);
ierr = PetscMalloc1(m + 1, &d_nnz);
CHKERRQ(ierr);
ierr = PetscMalloc1(m + 1, &o_nnz);
CHKERRQ(ierr);
for (i = Ni0, ic = 0; i < Ni1; i++) {
for (j = Nj0; j < Nj1; j++) {
for (k = Nk0; k < Nk1; k++) {
nbc = 0;
if (i == Ni0 || i == Ni1 - 1) nbc++;
if (j == Nj0 || j == Nj1 - 1) nbc++;
if (k == Nk0 || k == Nk1 - 1) nbc++;
for (jj = 0; jj < 3; jj++, ic++) {
d_nnz[ic] = 3 * (27 - osz[nbc]);
o_nnz[ic] = 3 * osz[nbc];
}
}
}
}
if (ic != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "ic %D does not equal m %D", ic, m);
ierr = MatCreate(PETSC_COMM_WORLD, &A);
CHKERRQ(ierr);
ierr = MatSetSizes(A, m, m, M, M);
CHKERRQ(ierr);
ierr = MatSetBlockSize(A, 3);
CHKERRQ(ierr);
ierr = MatSetType(A, MATAIJ);
CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A, 0, d_nnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A, 0, d_nnz, 0, o_nnz);
CHKERRQ(ierr);
ierr = PetscFree(d_nnz);
CHKERRQ(ierr);
ierr = PetscFree(o_nnz);
CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A, &Istart, &Iend);
CHKERRQ(ierr);
if (m != Iend - Istart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "m %D does not equal Iend %D - Istart %D", m, Iend, Istart);
ierr = VecCreate(PETSC_COMM_WORLD, &x);
CHKERRQ(ierr);
ierr = VecSetSizes(x, m, M);
CHKERRQ(ierr);
ierr = VecSetBlockSize(x, 3);
CHKERRQ(ierr);
ierr = VecSetFromOptions(x);
CHKERRQ(ierr);
ierr = VecDuplicate(x, &rhs);
CHKERRQ(ierr);
ierr = PetscMalloc1(m + 1, &coords);
CHKERRQ(ierr);
coords[m] = -99.0;
PetscReal h = 1.0 / ne;
for (i = Ni0, ic = 0; i < Ni1; i++) {
for (j = Nj0; j < Nj1; j++) {
for (k = Nk0; k < Nk1; k++, ic++) {
coords[3 * ic] = h * (PetscReal)i;
coords[3 * ic + 1] = h * (PetscReal)j;
coords[3 * ic + 2] = h * (PetscReal)k;
}
}
}
}
PetscReal s_r[SIZE_ARRAY_R] = {30, 0.1, 20, 10};
PetscReal x_r[SIZE_ARRAY_R] = {0.5, 0.4, 0.4, 0.4};
PetscReal y_r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4};
PetscReal z_r[SIZE_ARRAY_R] = {0.5, 0.45, 0.4, 0.35};
PetscReal r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4};
AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
MatNullSpace matnull;
Vec vec_coords;
PetscScalar* c;
ierr = VecCreate(MPI_COMM_WORLD, &vec_coords);
CHKERRQ(ierr);
ierr = VecSetBlockSize(vec_coords, 3);
CHKERRQ(ierr);
ierr = VecSetSizes(vec_coords, m, PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecSetUp(vec_coords);
CHKERRQ(ierr);
ierr = VecGetArray(vec_coords, &c);
CHKERRQ(ierr);
for (i = 0; i < m; i++) c[i] = coords[i];
ierr = VecRestoreArray(vec_coords, &c);
CHKERRQ(ierr);
ierr = MatNullSpaceCreateRigidBody(vec_coords, &matnull);
CHKERRQ(ierr);
ierr = MatSetNearNullSpace(A, matnull);
CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&matnull);
CHKERRQ(ierr);
ierr = VecDestroy(&vec_coords);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
double time = MPI_Wtime();
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime() - time;
ierr = PetscPrintf(PETSC_COMM_WORLD, "--- PC setup = %f\n", time);
CHKERRQ(ierr);
float t_time[SIZE_ARRAY_R];
int t_its[SIZE_ARRAY_R];
{
{
ierr = KSPSolve(ksp, rhs, x);
CHKERRQ(ierr);
ierr = KSPReset(ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = VecZeroEntries(x);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime();
ierr = KSPSolve(ksp, rhs, x);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
t_time[i] = MPI_Wtime() - time;
PetscInt its;
ierr = KSPGetIterationNumber(ksp, &its);
CHKERRQ(ierr);
t_its[i] = its;
ierr = ComputeError(A, rhs, x);
CHKERRQ(ierr);
if (i == (SIZE_ARRAY_R - 1))
AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m);
else
AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]);
CHKERRQ(ierr);
if (i > 0) {
t_its[0] += t_its[i];
t_time[0] += t_time[i];
}
}
if (SIZE_ARRAY_R > 1) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]);
CHKERRQ(ierr);
}
}
{
ierr = KSPGetPC(ksp, &pc);
CHKERRQ(ierr);
HpddmCustomOperator H;
H._A = A;
H._M = pc;
H._mv = mv;
H._precond = precond;
H._b = rhs;
H._x = x;
int n;
MatGetLocalSize(A, &n, NULL);
{
ierr = VecZeroEntries(x);
K* pt_rhs;
K* pt_x;
VecGetArray(rhs, &pt_rhs);
VecGetArray(x, &pt_x);
int previous = HpddmOptionVal(opt, "verbosity");
if (previous > 0) HpddmOptionRemove(opt, "verbosity");
HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD);
if (previous > 0) {
char buffer[20];
snprintf(buffer, 20, "%d", previous);
char* concat = malloc(strlen("-hpddm_verbosity ") + strlen(buffer) + 1);
strcpy(concat, "-hpddm_verbosity ");
strcat(concat, buffer);
HpddmOptionParseString(opt, concat);
free(concat);
}
VecRestoreArray(x, &pt_x);
VecRestoreArray(rhs, &pt_rhs);
previous = HpddmOptionVal(opt, "krylov_method");
if(previous == 4 || previous == 5) HpddmDestroyRecycling();
ierr = KSPReset(ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = VecZeroEntries(x);
CHKERRQ(ierr);
K* pt_rhs;
K* pt_x;
VecGetArray(rhs, &pt_rhs);
VecGetArray(x, &pt_x);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime();
t_its[i] = HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD);
MPI_Barrier(PETSC_COMM_WORLD);
t_time[i] = MPI_Wtime() - time;
VecRestoreArray(x, &pt_x);
VecRestoreArray(rhs, &pt_rhs);
ierr = ComputeError(A, rhs, x);
CHKERRQ(ierr);
if (i != (SIZE_ARRAY_R - 1)) {
AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]);
CHKERRQ(ierr);
if (i > 0) {
t_its[0] += t_its[i];
t_time[0] += t_time[i];
}
}
if (SIZE_ARRAY_R > 1) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]);
CHKERRQ(ierr);
}
}
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&rhs);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = PetscFree(coords);
CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
MatNullSpaceCreateRigidBody - create rigid body modes from coordinates
Collective on Vec
Input Argument:
. coords - block of coordinates of each node, must have block size set
Output Argument:
. sp - the null space
Level: advanced
.seealso: MatNullSpaceCreate()
@*/
PetscErrorCode MatNullSpaceCreateRigidBody(Vec coords,MatNullSpace *sp)
{
PetscErrorCode ierr;
const PetscScalar *x;
PetscScalar *v[6],dots[3];
Vec vec[6];
PetscInt n,N,dim,nmodes,i,j;
PetscFunctionBegin;
ierr = VecGetBlockSize(coords,&dim);CHKERRQ(ierr);
ierr = VecGetLocalSize(coords,&n);CHKERRQ(ierr);
ierr = VecGetSize(coords,&N);CHKERRQ(ierr);
n /= dim;
N /= dim;
switch (dim) {
case 1:
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_TRUE,0,NULL,sp);CHKERRQ(ierr);
break;
case 2:
case 3:
nmodes = (dim == 2) ? 3 : 6;
ierr = VecCreate(PetscObjectComm((PetscObject)coords),&vec[0]);CHKERRQ(ierr);
ierr = VecSetSizes(vec[0],dim*n,dim*N);CHKERRQ(ierr);
ierr = VecSetBlockSize(vec[0],dim);CHKERRQ(ierr);
ierr = VecSetUp(vec[0]);CHKERRQ(ierr);
for (i=1; i<nmodes; i++) {ierr = VecDuplicate(vec[0],&vec[i]);CHKERRQ(ierr);}
for (i=0; i<nmodes; i++) {ierr = VecGetArray(vec[i],&v[i]);CHKERRQ(ierr);}
ierr = VecGetArrayRead(coords,&x);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (dim == 2) {
v[0][i*2+0] = 1./N;
v[0][i*2+1] = 0.;
v[1][i*2+0] = 0.;
v[1][i*2+1] = 1./N;
/* Rotations */
v[2][i*2+0] = -x[i*2+1];
v[2][i*2+1] = x[i*2+0];
} else {
v[0][i*3+0] = 1./N;
v[0][i*3+1] = 0.;
v[0][i*3+2] = 0.;
v[1][i*3+0] = 0.;
v[1][i*3+1] = 1./N;
v[1][i*3+2] = 0.;
v[2][i*3+0] = 0.;
v[2][i*3+1] = 0.;
v[2][i*3+2] = 1./N;
v[3][i*3+0] = x[i*3+1];
v[3][i*3+1] = -x[i*3+0];
v[3][i*3+2] = 0.;
v[4][i*3+0] = 0.;
v[4][i*3+1] = -x[i*3+2];
v[4][i*3+2] = x[i*3+1];
v[5][i*3+0] = x[i*3+2];
v[5][i*3+1] = 0.;
v[5][i*3+2] = -x[i*3+0];
}
}
for (i=0; i<nmodes; i++) {ierr = VecRestoreArray(vec[i],&v[i]);CHKERRQ(ierr);}
ierr = VecRestoreArrayRead(coords,&x);CHKERRQ(ierr);
for (i=dim; i<nmodes; i++) {
/* Orthonormalize vec[i] against vec[0:dim] */
ierr = VecMDot(vec[i],i,vec,dots);CHKERRQ(ierr);
for (j=0; j<i; j++) dots[j] *= -1.;
ierr = VecMAXPY(vec[i],i,dots,vec);CHKERRQ(ierr);
ierr = VecNormalize(vec[i],NULL);CHKERRQ(ierr);
}
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_FALSE,nmodes,vec,sp);CHKERRQ(ierr);
for (i=0; i<nmodes; i++) {ierr = VecDestroy(&vec[i]);CHKERRQ(ierr);}
}
PetscFunctionReturn(0);
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file viewer
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
and http://www.geuz.org/gmsh/doc/texinfo/#MSH-binary-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscViewerType vtype;
GmshElement *gmsh_elem;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d, r, cell;
int i, numVertices = 0, numCells = 0, trueNumCells = 0, numRegions = 0, snum;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match, binary, bswap = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
ierr = PetscViewerGetType(viewer, &vtype);CHKERRQ(ierr);
ierr = PetscStrcmp(vtype, PETSCVIEWERBINARY, &binary);CHKERRQ(ierr);
if (!rank || binary) {
PetscBool match;
int fileType, dataSize;
float version;
/* Read header */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$MeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 3, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%f %d %d", &version, &fileType, &dataSize);
if (snum != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Gmsh file header: %s", line);
if (version < 2.0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Gmsh file must be at least version 2.0");
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
if (binary) {
int checkInt;
ierr = PetscViewerRead(viewer, &checkInt, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
if (checkInt != 1) {
ierr = PetscByteSwap(&checkInt, PETSC_ENUM, 1);CHKERRQ(ierr);
if (checkInt == 1) bswap = PETSC_TRUE;
else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh binary file", fileType);
}
} else if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndMeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* OPTIONAL Read physical names */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$PhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (match) {
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numRegions);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
for (r = 0; r < numRegions; ++r) {
PetscInt rdim, tag;
ierr = PetscViewerRead(viewer, &rdim, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &tag, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndPhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Initial read for vertex section */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
/* Read vertices */
ierr = PetscStrncmp(line, "$Nodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numVertices);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscMalloc1(numVertices*3, &coordsIn);CHKERRQ(ierr);
if (binary) {
size_t doubleSize, intSize;
PetscInt elementSize;
char *buffer;
PetscScalar *baseptr;
ierr = PetscDataTypeGetSize(PETSC_ENUM, &intSize);CHKERRQ(ierr);
ierr = PetscDataTypeGetSize(PETSC_DOUBLE, &doubleSize);CHKERRQ(ierr);
elementSize = (intSize + 3*doubleSize);
ierr = PetscMalloc1(elementSize*numVertices, &buffer);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, buffer, elementSize*numVertices, NULL, PETSC_CHAR);CHKERRQ(ierr);
if (bswap) ierr = PetscByteSwap(buffer, PETSC_CHAR, elementSize*numVertices);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
baseptr = ((PetscScalar*)(buffer+v*elementSize+intSize));
coordsIn[v*3+0] = baseptr[0];
coordsIn[v*3+1] = baseptr[1];
coordsIn[v*3+2] = baseptr[2];
}
ierr = PetscFree(buffer);CHKERRQ(ierr);
} else {
for (v = 0; v < numVertices; ++v) {
ierr = PetscViewerRead(viewer, &i, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &(coordsIn[v*3]), 3, NULL, PETSC_DOUBLE);CHKERRQ(ierr);
if (i != (int)v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndNodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$Elements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
snum = sscanf(line, "%d", &numCells);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
if (!rank || binary) {
/* Gmsh elements can be of any dimension/co-dimension, so we need to traverse the
file contents multiple times to figure out the true number of cells and facets
in the given mesh. To make this more efficient we read the file contents only
once and store them in memory, while determining the true number of cells. */
ierr = DMPlexCreateGmsh_ReadElement(viewer, numCells, binary, bswap, &gmsh_elem);CHKERRQ(ierr);
for (trueNumCells=0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim > dim) {dim = gmsh_elem[c].dim; trueNumCells = 0;}
if (gmsh_elem[c].dim == dim) trueNumCells++;
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndElements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
/* For binary we read on all ranks, but only build the plex on rank 0 */
if (binary && rank) {trueNumCells = 0; numVertices = 0;};
/* Allocate the cell-vertex mesh */
ierr = DMPlexSetChart(*dm, 0, trueNumCells+numVertices);CHKERRQ(ierr);
if (!rank) {
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
ierr = DMPlexSetConeSize(*dm, cell, gmsh_elem[c].numNodes);CHKERRQ(ierr);
cell++;
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
/* Add cell-vertex connections */
if (!rank) {
PetscInt pcone[8], corner;
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + trueNumCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (gmsh_elem[c].numNodes == 4) {
PetscInt tmp = pcone[0];
pcone[0] = pcone[1];
pcone[1] = tmp;
}
/* Hexahedra are inverted */
if (gmsh_elem[c].numNodes == 8) {
PetscInt tmp = pcone[1];
pcone[1] = pcone[3];
pcone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, cell, pcone);CHKERRQ(ierr);
cell++;
}
}
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
/* Apply boundary IDs by finding the relevant facets with vertex joins */
PetscInt pcone[8], corner, vStart, vEnd;
ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim-1) {
PetscInt joinSize;
const PetscInt *join;
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + vStart - 1;
}
ierr = DMPlexGetFullJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for element %d", gmsh_elem[c].id);
ierr = DMSetLabelValue(*dm, "Face Sets", join[0], gmsh_elem[c].tags[0]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
}
}
/* Create cell sets */
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
if (gmsh_elem[c].numTags > 0) {
ierr = DMSetLabelValue(*dm, "Cell Sets", cell, gmsh_elem[c].tags[0]);CHKERRQ(ierr);
cell++;
}
}
}
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, trueNumCells, trueNumCells + numVertices);CHKERRQ(ierr);
for (v = trueNumCells; v < trueNumCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Clean up intermediate storage */
if (!rank || binary) ierr = PetscFree(gmsh_elem);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints)
{
MPI_Comm comm = ctx->comm;
PetscScalar *a;
PetscInt p, q, i;
PetscMPIInt rank, size;
PetscErrorCode ierr;
Vec pointVec;
IS cellIS;
PetscLayout layout;
PetscReal *globalPoints;
PetscScalar *globalPointsScalar;
const PetscInt *ranges;
PetscMPIInt *counts, *displs;
const PetscInt *foundCells;
PetscMPIInt *foundProcs, *globalProcs;
PetscInt n, N;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set");
/* Locate points */
n = ctx->nInput;
if (!redundantPoints) {
ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr);
/* Communicate all points to all processes */
ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr);
ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
counts[p] = (ranges[p+1] - ranges[p])*ctx->dim;
displs[p] = ranges[p]*ctx->dim;
}
ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr);
} else {
N = n;
globalPoints = ctx->points;
counts = displs = NULL;
layout = NULL;
}
#if 0
ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
/* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */
#else
#if defined(PETSC_USE_COMPLEX)
ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr);
for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i];
#else
globalPointsScalar = globalPoints;
#endif
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr);
ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr);
ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr);
#endif
for (p = 0; p < N; ++p) {
if (foundCells[p] >= 0) foundProcs[p] = rank;
else foundProcs[p] = size;
}
/* Let the lowest rank process own each point */
ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr);
ctx->n = 0;
for (p = 0; p < N; ++p) {
if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0);
else if (globalProcs[p] == rank) ctx->n++;
}
/* Create coordinates vector and array of owned cells */
ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr);
ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr);
ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr);
ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr);
for (p = 0, q = 0, i = 0; p < N; ++p) {
if (globalProcs[p] == rank) {
PetscInt d;
for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d];
ctx->cells[q++] = foundCells[p];
}
}
ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr);
#if 0
ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr);
#else
ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr);
ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr);
ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
#endif
if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);}
if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);}
ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode SNESSolve_Multiblock(SNES snes)
{
SNES_Multiblock *mb = (SNES_Multiblock *) snes->data;
Vec X, Y, F;
PetscReal fnorm;
PetscInt maxits, i;
PetscErrorCode ierr;
PetscFunctionBegin;
snes->reason = SNES_CONVERGED_ITERATING;
maxits = snes->max_its; /* maximum number of iterations */
X = snes->vec_sol; /* X^n */
Y = snes->vec_sol_update; /* \tilde X */
F = snes->vec_func; /* residual vector */
ierr = VecSetBlockSize(X, mb->bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(Y, mb->bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(F, mb->bs);CHKERRQ(ierr);
ierr = PetscObjectTakeAccess(snes);CHKERRQ(ierr);
snes->iter = 0;
snes->norm = 0.;
ierr = PetscObjectGrantAccess(snes);CHKERRQ(ierr);
if (!snes->vec_func_init_set){
ierr = SNESComputeFunction(snes, X, F);CHKERRQ(ierr);
if (snes->domainerror) {
snes->reason = SNES_DIVERGED_FUNCTION_DOMAIN;
PetscFunctionReturn(0);
}
} else {
snes->vec_func_init_set = PETSC_FALSE;
}
if (!snes->norm_init_set) {
ierr = VecNorm(F, NORM_2, &fnorm);CHKERRQ(ierr); /* fnorm <- ||F|| */
if (PetscIsInfOrNanReal(fnorm)) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_FP, "Infinite or not-a-number generated in norm");
} else {
fnorm = snes->norm_init;
snes->norm_init_set = PETSC_FALSE;
}
ierr = PetscObjectTakeAccess(snes);CHKERRQ(ierr);
snes->norm = fnorm;
ierr = PetscObjectGrantAccess(snes);CHKERRQ(ierr);
SNESLogConvHistory(snes,fnorm,0);
ierr = SNESMonitor(snes,0,fnorm);CHKERRQ(ierr);
/* set parameter for default relative tolerance convergence test */
snes->ttol = fnorm*snes->rtol;
/* test convergence */
ierr = (*snes->ops->converged)(snes,0,0.0,0.0,fnorm,&snes->reason,snes->cnvP);CHKERRQ(ierr);
if (snes->reason) PetscFunctionReturn(0);
for (i = 0; i < maxits; i++) {
/* Call general purpose update function */
if (snes->ops->update) {
ierr = (*snes->ops->update)(snes, snes->iter);CHKERRQ(ierr);
}
/* Compute X^{new} from subsolves */
if (mb->type == PC_COMPOSITE_ADDITIVE) {
BlockDesc blocks = mb->blocks;
if (mb->defaultblocks) {
/*TODO: Make an array of Vecs for this */
/*ierr = VecStrideGatherAll(X, mb->x, INSERT_VALUES);CHKERRQ(ierr);*/
while (blocks) {
ierr = SNESSolve(blocks->snes, PETSC_NULL, blocks->x);CHKERRQ(ierr);
blocks = blocks->next;
}
/*ierr = VecStrideScatterAll(mb->x, X, INSERT_VALUES);CHKERRQ(ierr);*/
} else {
while (blocks) {
ierr = VecScatterBegin(blocks->sctx, X, blocks->x, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(blocks->sctx, X, blocks->x, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
ierr = SNESSolve(blocks->snes, PETSC_NULL, blocks->x);CHKERRQ(ierr);
ierr = VecScatterBegin(blocks->sctx, blocks->x, X, INSERT_VALUES, SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(blocks->sctx, blocks->x, X, INSERT_VALUES, SCATTER_REVERSE);CHKERRQ(ierr);
blocks = blocks->next;
}
}
} else {
SETERRQ1(((PetscObject) snes)->comm, PETSC_ERR_SUP, "Unsupported or unknown composition", (int) mb->type);
}
CHKMEMQ;
/* Compute F(X^{new}) */
ierr = SNESComputeFunction(snes, X, F);CHKERRQ(ierr);
ierr = VecNorm(F, NORM_2, &fnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(fnorm)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_FP,"Infinite or not-a-number generated norm");
if (snes->nfuncs >= snes->max_funcs) {
snes->reason = SNES_DIVERGED_FUNCTION_COUNT;
break;
}
if (snes->domainerror) {
snes->reason = SNES_DIVERGED_FUNCTION_DOMAIN;
PetscFunctionReturn(0);
}
/* Monitor convergence */
ierr = PetscObjectTakeAccess(snes);CHKERRQ(ierr);
snes->iter = i+1;
snes->norm = fnorm;
ierr = PetscObjectGrantAccess(snes);CHKERRQ(ierr);
SNESLogConvHistory(snes,snes->norm,0);
ierr = SNESMonitor(snes,snes->iter,snes->norm);CHKERRQ(ierr);
/* Test for convergence */
ierr = (*snes->ops->converged)(snes,snes->iter,0.0,0.0,fnorm,&snes->reason,snes->cnvP);CHKERRQ(ierr);
if (snes->reason) break;
}
if (i == maxits) {
ierr = PetscInfo1(snes, "Maximum number of iterations has been reached: %D\n", maxits);CHKERRQ(ierr);
if (!snes->reason) snes->reason = SNES_DIVERGED_MAX_IT;
}
PetscFunctionReturn(0);
}
Solver::Petsc::Petsc (Grid& gr)
{
world = PETSC_COMM_WORLD;
MPI_Comm_rank (world, &rank);
MPI_Comm_size (world, &nProcs);
n = gr.n_in_elm;
bs = N_VAR;
vecGlobalSize = n*bs;
//DX = (double*) malloc (vecGlobalSize*sizeof(double));
//PetscMalloc1 (xGlobalSize, &DX);
DX = new double [vecGlobalSize];
// set x
VecCreate (world, &x);
VecSetType (x, VECSTANDARD);
VecSetBlockSize(x, bs);
VecSetSizes (x, PETSC_DECIDE, vecGlobalSize);
VecGetLocalSize (x, &vecLocalSize);
VecGetOwnershipRange (x, &vecLocBeg, &vecLocEnd);
// set b
VecDuplicate (x, &b);
// set A
MatCreate (world, &A);
MatSetType (A, MATMPIBAIJ);
//MatSetType (A, MATMPIAIJ);
MatSetSizes (A, PETSC_DECIDE, PETSC_DECIDE, vecGlobalSize, vecGlobalSize);
//MatSeqBAIJSetPreallocation (A, bs, 4, NULL);
//MatSeqAIJSetPreallocation (A, 4*bs, NULL);
//MatMPIAIJSetPreallocation (A, 4*bs, NULL, 4*bs, NULL);
MatMPIBAIJSetPreallocation (A, bs, 5, NULL, 5, NULL);
//MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
//MatCreateBAIJ (world, bs, PETSC_DECIDE, PETSC_DECIDE, n*bs, n*bs, 1, NULL, 3, NULL, &A);
// specific to pentagonal mesh . change later
MatGetOwnershipRange (A, &matLocBeg, &matLocEnd);
MatGetLocalSize (A, &matLocalSize, NULL);
KSPCreate (world, &ksp);
KSPSetOperators (ksp, A, A);
KSPGetPC (ksp, &pc);
PCSetType (pc, PCSOR);
KSPSetType (ksp, KSPGMRES);
KSPSetFromOptions (ksp);
localSizes = new int [nProcs];
int recvcounts[nProcs];
int displs[nProcs];
displs[0] = 0;
for (int i=0; i<nProcs; ++i)
{
recvcounts[i] = 1;
}
for (int i=1; i<nProcs; ++i)
{
displs[i] = displs[i-1] + recvcounts[i-1];
}
MPI_Allgatherv (&vecLocalSize, 1, MPI_INT, localSizes, recvcounts, displs, MPI_INT, world);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
SNES snes; /* nonlinear solver */
Vec Hu; /* solution vector */
AppCtx user; /* user-defined work context */
ExactCtx exact;
PetscInt its; /* snes reports iteration count */
SNESConvergedReason reason; /* snes reports convergence */
PetscReal tmp1, tmp2, tmp3,
errnorms[2], scaleNode[2], descaleNode[2];
PetscInt i;
char dumpfile[80],dxdocstr[80];
PetscBool eps_set = PETSC_FALSE,
dump = PETSC_FALSE,
exactinitial = PETSC_FALSE,
snes_mf_set, snes_fd_set, dx_set;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &user.rank); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"MARINE solves for thickness and velocity in 1D, steady marine ice sheet\n"
" [run with -help for info and options]\n");CHKERRQ(ierr);
user.n = 3.0; /* Glen flow law exponent */
user.secpera = 31556926.0;
user.rho = 910.0; /* kg m^-3 */
user.rhow = 1028.0; /* kg m^-3 */
user.omega = 1.0 - user.rho / user.rhow;
user.g = 9.81; /* m s^-2 */
/* get parameters of exact solution */
ierr = params_exactBod(&tmp1, &(exact.L0), &(exact.xg), &tmp2, &tmp3, &(user.k)); CHKERRQ(ierr);
ierr = exactBod(exact.xg,&(exact.Hg),&tmp2,&(exact.Mg)); CHKERRQ(ierr);
ierr = exactBodBueler(exact.xg,&tmp1,&(exact.Bg)); CHKERRQ(ierr);
user.zocean = user.rho * exact.Hg / user.rhow;
/* see ../marineshoot.py: */
#define xa_default 0.2
#define xc_default 0.98
/* define interval [xa,xc] */
user.xa = xa_default * exact.L0;
user.xc = xc_default * exact.L0;
/* get Dirichlet boundary conditions, and mass balance on shelf */
ierr = exactBod(user.xa, &(user.Ha), &(user.ua), &tmp1); CHKERRQ(ierr);
/* regularize using strain rate of 1/(length) per year */
user.epsilon = (1.0 / user.secpera) / (user.xc - user.xa);
/*user.epsilon = 0.0;*/
user.Hscale = 1000.0;
user.uscale = 100.0 / user.secpera;
user.noscale = PETSC_FALSE;
user.dx = 10000.0; /* default to coarse 10 km grid */
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,
"","options to marine (steady marine ice sheet solver)","");CHKERRQ(ierr);
{
ierr = PetscOptionsBool("-snes_mf","","",PETSC_FALSE,&snes_mf_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-snes_fd","","",PETSC_FALSE,&snes_fd_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-noscale","","",PETSC_FALSE,&user.noscale,NULL);CHKERRQ(ierr);
snprintf(dxdocstr,80,"target grid spacing (m) on interval of length %.0f m",
user.xc-user.xa);
ierr = PetscOptionsReal("-dx",dxdocstr,"",user.dx,&user.dx,&dx_set);CHKERRQ(ierr);
ierr = PetscOptionsBool("-exactinit",
"initialize using exact solution instead of default linear function","",
PETSC_FALSE,&exactinitial,NULL);CHKERRQ(ierr);
ierr = PetscOptionsString("-dump",
"dump approx and exact solution into given file (as ascii matlab format)","",
NULL,dumpfile,80,&dump);CHKERRQ(ierr);
ierr = PetscOptionsReal("-epsilon","regularizing strain rate for stress computation (a-1)","",
user.epsilon * user.secpera,&user.epsilon,&eps_set);CHKERRQ(ierr);
if (eps_set) user.epsilon *= 1.0 / user.secpera;
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
if (snes_fd_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" using approximate Jacobian; finite-differencing using coloring\n");
CHKERRQ(ierr);
} else if (snes_mf_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" matrix free; no preconditioner\n"); CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" true Jacobian\n"); CHKERRQ(ierr);
}
if (dx_set && (user.dx <= 0.0)) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***ERROR: -dx value must be positive ... USAGE FOLLOWS ...\n\n%s",help);
PetscEnd();
}
user.N = (int)ceil( ((user.xc - user.xa) / user.dx) - 0.5 );
user.Mx = user.N + 2;
user.dx = (user.xc - user.xa) / ((PetscReal)(user.N) + 0.5); /* recompute so dx * (N+1/2) = xc - xa */
if (dx_set && (user.dx < 1.0)) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***WARNING: '-dx %.3f' meters is below one meter and creates grid of %d points\n"
" ... probably uncomputable!\n\n",
user.dx,user.Mx);
}
/* residual scaling coeffs; motivation at right */
user.rscHa = 1.0 / user.Hscale, /* Dirichlet cond for H */
user.rscua = 1.0 / user.uscale, /* Dirichlet cond for u */
user.rscuH = user.dx / (user.Hscale * user.uscale), /* flux derivative d(uH)/dx */
user.rscstress = 1.0 / (user.k * user.rho * user.g * user.Hscale * user.uscale), /* beta term in SSA */
user.rsccalv = 1.0 / (0.025 * user.rho * user.g * user.Hscale); /* 0.5 * omega * overburden */
/* Create machinery for parallel grid management (DMDA), nonlinear solver (SNES),
and Vecs for fields (solution, RHS). Degrees of freedom = 2 (thickness and
velocity at each point). */
ierr = DMDACreate1d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,user.Mx,2,1,PETSC_NULL,&user.da);
CHKERRQ(ierr);
ierr = DMSetApplicationContext(user.da,&user);CHKERRQ(ierr);
ierr = DMDASetFieldName(user.da,0,"ice thickness [non-dimensional]"); CHKERRQ(ierr);
ierr = DMDASetFieldName(user.da,1,"ice velocity [non-dimensional]"); CHKERRQ(ierr);
ierr = DMSetFromOptions(user.da); CHKERRQ(ierr);
ierr = DMDAGetInfo(user.da,PETSC_IGNORE,&user.Mx,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);CHKERRQ(ierr);
ierr = DMDAGetCorners(user.da,&user.xs,PETSC_NULL,PETSC_NULL,&user.xm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
/* another DMDA for scalar staggered parameters; one fewer point */
ierr = DMDACreate1d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,user.Mx-1,1,1,PETSC_NULL,&user.stagda);
CHKERRQ(ierr);
/* establish geometry on grid; note xa = x_0 and xc = x_{N+1/2} */
ierr = DMDASetUniformCoordinates(user.da, user.xa, user.xc+0.5*user.dx,
0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(user.stagda,user.xa+0.5*user.dx,user.xc,
0.0,1.0,0.0,1.0);CHKERRQ(ierr);
/* report on current grid */
ierr = PetscPrintf(PETSC_COMM_WORLD,
" grid: Mx = N+2 = %D regular points, dx = %.3f m, xa = %.2f km, xc = %.2f km\n",
user.Mx, user.dx, user.xa/1000.0, user.xc/1000.0);CHKERRQ(ierr);
/* Extract/allocate global vectors from DMDAs and duplicate for remaining same types */
ierr = DMCreateGlobalVector(user.da,&Hu);CHKERRQ(ierr);
ierr = VecSetBlockSize(Hu,2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)Hu,"Hu");CHKERRQ(ierr);
ierr = VecDuplicate(Hu,&exact.Hu);CHKERRQ(ierr); /* inherits block size */
ierr = PetscObjectSetName((PetscObject)exact.Hu,"exactHu");CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user.stagda,&user.Mstag);CHKERRQ(ierr);
ierr = VecDuplicate(user.Mstag,&user.Bstag);CHKERRQ(ierr);
/* set up snes */
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes,user.da);CHKERRQ(ierr);
ierr = DMDASNESSetFunctionLocal(user.da,INSERT_VALUES,(DMDASNESFunction)scshell,&user);CHKERRQ(ierr);
ierr = DMDASNESSetJacobianLocal(user.da,(DMDASNESJacobian)JacobianMatrixLocal,&user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* the exact thickness and exact ice velocity (user.uHexact) are known */
ierr = FillExactSoln(&exact, &user); CHKERRQ(ierr);
/* the exact solution allows setting M(x), B(x) */
ierr = FillDistributedParams(&exact, &user);CHKERRQ(ierr);
if (exactinitial) {
ierr = PetscPrintf(PETSC_COMM_WORLD," using exact solution as initial guess\n"); CHKERRQ(ierr);
/* the initial guess is the exact continuum solution */
ierr = VecCopy(exact.Hu,Hu); CHKERRQ(ierr);
} else {
/* the initial guess is a linear solution */
ierr = FillInitial(&user, &Hu); CHKERRQ(ierr);
}
/************ SOLVE NONLINEAR SYSTEM ************/
/* recall that RHS r is used internally by KSP, and is set by the SNES */
if (user.noscale) {
user.Hscale = 1.0;
user.uscale = 1.0;
}
scaleNode[0] = user.Hscale;
scaleNode[1] = user.uscale;
for (i = 0; i < 2; i++) descaleNode[i] = 1.0 / scaleNode[i];
ierr = VecStrideScaleAll(Hu,descaleNode); CHKERRQ(ierr); /* de-dimensionalize initial guess */
ierr = SNESSolve(snes,PETSC_NULL,Hu);CHKERRQ(ierr);
ierr = VecStrideScaleAll(Hu,scaleNode); CHKERRQ(ierr); /* put back in "real" scale */
/* minimal report on solve */
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = SNESGetConvergedReason(snes,&reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" %s Number of Newton iterations = %D\n",
SNESConvergedReasons[reason],its);CHKERRQ(ierr);
if (dump) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
"dumping results in ascii matlab format to file '%s' ...\n",dumpfile);CHKERRQ(ierr);
PetscViewer viewer;
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,dumpfile,&viewer);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);CHKERRQ(ierr);
DM coord_da;
Vec coord_x;
ierr = DMGetCoordinateDM(user.da, &coord_da); CHKERRQ(ierr);
ierr = DMGetCoordinates(user.da, &coord_x); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% START MATLAB\n"); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)coord_x,"x");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% viewing coordinate vector x \n");CHKERRQ(ierr);
ierr = VecView(coord_x,viewer); CHKERRQ(ierr);
ierr = VecView(Hu,viewer); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% viewing combined result Hu\n");CHKERRQ(ierr);
ierr = VecView(Hu,viewer); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% viewing combined exact result exactHu\n");CHKERRQ(ierr);
ierr = VecView(exact.Hu,viewer); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,
"%% defining plottable variables and plotting in Matlab\n"
"Mx = %d; secpera = 31556926.0;\n"
"H = Hu(1:2:2*Mx-1);\n"
"u = Hu(2:2:2*Mx);\n"
"exactH = exactHu(1:2:2*Mx-1);\n"
"exactu = exactHu(2:2:2*Mx);\n"
"figure, plot(x,H,x,exactH);\n"
"legend('numerical','exact'), xlabel x, ylabel('H (m)')\n"
"figure, plot(x,u*secpera,x,exactu*secpera);\n"
"legend('numerical','exact'), xlabel x, ylabel('u (m/a)')\n"
"figure, subplot(2,1,1), semilogy(x,abs(H-exactH));\n"
"ylabel('H error (m)'), grid\n"
"subplot(2,1,2), semilogy(x,abs(u-exactu)*secpera);\n"
"xlabel x, ylabel('u error (m/a)'), grid\n",
user.Mx); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% END MATLAB\n"); CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
/* evaluate error relative to exact solution */
ierr = VecAXPY(Hu,-1.0,exact.Hu); CHKERRQ(ierr); /* Hu = - Huexact + Hu */
ierr = VecStrideNormAll(Hu,NORM_INFINITY,errnorms); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"(dx,errHinf,erruinf) %.3f %.4e %.4e\n",
user.dx,errnorms[0],errnorms[1]*user.secpera);CHKERRQ(ierr);
ierr = VecDestroy(&Hu);CHKERRQ(ierr);
ierr = VecDestroy(&(exact.Hu));CHKERRQ(ierr);
ierr = VecDestroy(&(user.Mstag));CHKERRQ(ierr);
ierr = VecDestroy(&(user.Bstag));CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&(user.da));CHKERRQ(ierr);
ierr = DMDestroy(&(user.stagda));CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
