// Write the sub mesh into a HDF5 file.
PetscErrorCode ProbeVolume::writeSubMeshHDF5(const std::string &filePath)
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
// only the first process in the communicator write the sub-mesh into a file
if (commRank == 0)
{
// because only one process is involved in writing the sub-mesh,
// we need to create a temporary viewer
PetscViewer viewer2;
ierr = PetscViewerCreate(PETSC_COMM_SELF, &viewer2); CHKERRQ(ierr);
ierr = PetscViewerSetType(viewer2, PETSCVIEWERHDF5); CHKERRQ(ierr);
ierr = PetscViewerFileSetMode(viewer2, FILE_MODE_WRITE); CHKERRQ(ierr);
ierr = PetscViewerFileSetName(
viewer2, filePath.c_str()); CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer2, "mesh"); CHKERRQ(ierr);
std::vector<std::string> dirs{"x", "y", "z"};
for (unsigned int d = 0; d < coord.size(); ++d)
{
Vec tmp;
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, 1, nPtsDir[d],
&coord[d][0], &tmp); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) tmp,
dirs[d].c_str()); CHKERRQ(ierr);
ierr = VecView(tmp, viewer2); CHKERRQ(ierr);
ierr = VecDestroy(&tmp); CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer2); CHKERRQ(ierr);
}
PetscFunctionReturn(0);
} // ProbeVolume::writeSubMeshHDF5
// Write vector data data into a HDF5 file.
PetscErrorCode ProbeVolume::writeVecHDF5(const Vec &vec, const PetscReal &t)
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
std::string field_str = type::fd2str[field];
ierr = PetscViewerHDF5PushGroup(viewer, field_str.c_str()); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) vec,
std::to_string(t).c_str()); CHKERRQ(ierr);
ierr = VecView(vec, viewer); CHKERRQ(ierr);
if (count != 0) // add number of time-steps accumulated as attribute
{
ierr = PetscViewerHDF5WriteAttribute(
viewer, ("/" + field_str + "/" + std::to_string(t)).c_str(), "count",
PETSC_INT, &count); CHKERRQ(ierr);
}
PetscFunctionReturn(0);
} // ProbeVolume::writeVecHDF5
int main(int argc,char **argv)
{
Vec x1, x2, y1, y2, y3, y4;
PetscViewer viewer;
PetscMPIInt rank;
PetscInt i, nlocal, n = 6;
PetscScalar *array;
PetscBool equal;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscInitialize(&argc, &argv, (char*) 0, help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL, "-n", &n, NULL);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD, &x1);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x1, "TestVec");CHKERRQ(ierr);
ierr = VecSetSizes(x1, PETSC_DECIDE, n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x1);CHKERRQ(ierr);
/* initialize x1 */
ierr = VecGetLocalSize(x1, &nlocal);CHKERRQ(ierr);
ierr = VecGetArray(x1, &array);CHKERRQ(ierr);
for (i = 0; i < nlocal; i++) array[i] = rank + 1;
ierr = VecRestoreArray(x1, &array);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD, &x2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x2, "TestVec2");CHKERRQ(ierr);
ierr = VecSetSizes(x2, PETSC_DECIDE, n);CHKERRQ(ierr);
ierr = VecSetBlockSize(x2, 2);CHKERRQ(ierr);
ierr = VecSetFromOptions(x2);CHKERRQ(ierr);
/* initialize x2 */
ierr = VecGetLocalSize(x2, &nlocal);CHKERRQ(ierr);
ierr = VecGetArray(x2, &array);CHKERRQ(ierr);
for (i = 0; i < nlocal; i++) array[i] = rank + 1;
ierr = VecRestoreArray(x2, &array);CHKERRQ(ierr);
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD, "ex19.h5", FILE_MODE_WRITE, &viewer);CHKERRQ(ierr);
ierr = VecView(x1, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer, "/testBlockSize");CHKERRQ(ierr);
ierr = VecView(x2, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer, "/testTimestep");CHKERRQ(ierr);
ierr = PetscViewerHDF5SetTimestep(viewer, 0);CHKERRQ(ierr);
ierr = VecView(x2, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5SetTimestep(viewer, 1);CHKERRQ(ierr);
ierr = VecView(x2, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD, &y1);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y1, "TestVec");CHKERRQ(ierr);
ierr = VecSetSizes(y1, PETSC_DECIDE, n);CHKERRQ(ierr);
ierr = VecSetFromOptions(y1);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&y2);CHKERRQ(ierr);
ierr = VecSetBlockSize(y2, 2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y2, "TestVec2");CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&y3);CHKERRQ(ierr);
ierr = VecSetBlockSize(y3, 2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y3, "TestVec2");CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&y4);CHKERRQ(ierr);
ierr = VecSetBlockSize(y4, 2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y4, "TestVec2");CHKERRQ(ierr);
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD, "ex19.h5", FILE_MODE_READ, &viewer);CHKERRQ(ierr);
ierr = VecLoad(y1, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer, "/testBlockSize");CHKERRQ(ierr);
ierr = VecLoad(y2, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer, "/testTimestep");CHKERRQ(ierr);
ierr = PetscViewerHDF5SetTimestep(viewer, 0);CHKERRQ(ierr);
ierr = VecLoad(y3, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5SetTimestep(viewer, 1);CHKERRQ(ierr);
ierr = VecLoad(y4, viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecEqual(x1, y1, &equal);CHKERRQ(ierr);
if (!equal) {
ierr = VecView(x1, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecView(y1, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB, "Error in HDF5 viewer");
}
ierr = VecEqual(x2, y2, &equal);CHKERRQ(ierr);
if (!equal) {
ierr = VecView(x2, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecView(y2, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB, "Error in HDF5 viewer");
}
ierr = VecDestroy(&x1);CHKERRQ(ierr);
ierr = VecDestroy(&x2);CHKERRQ(ierr);
ierr = VecDestroy(&y1);CHKERRQ(ierr);
ierr = VecDestroy(&y2);CHKERRQ(ierr);
ierr = VecDestroy(&y3);CHKERRQ(ierr);
ierr = VecDestroy(&y4);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/**
* output
* ------
* Output the E and H fields.
*/
PetscErrorCode output(char *output_name, const Vec x, const Mat CF, const Vec conjParam, const Vec conjSrc, const GridInfo gi)
{
PetscFunctionBegin;
PetscErrorCode ierr;
char output_name_prefixed[PETSC_MAX_PATH_LEN];
//const char *prefix = "/out/";
const char *h_extension = ".H.h5";
const char *e_extension = ".E.h5";
//ierr = PetscStrcpy(output_name_prefixed, getenv("FD3D_ROOT")); CHKERRQ(ierr);
//ierr = PetscStrcat(output_name_prefixed, prefix); CHKERRQ(ierr);
//ierr = PetscStrcat(output_name_prefixed, output_name); CHKERRQ(ierr);
ierr = PetscStrcpy(output_name_prefixed, output_name); CHKERRQ(ierr);
char h_file[PETSC_MAX_PATH_LEN];
char e_file[PETSC_MAX_PATH_LEN];
ierr = PetscStrcpy(h_file, output_name_prefixed); CHKERRQ(ierr);
ierr = PetscStrcat(h_file, h_extension); CHKERRQ(ierr);
ierr = PetscStrcpy(e_file, output_name_prefixed); CHKERRQ(ierr);
ierr = PetscStrcat(e_file, e_extension); CHKERRQ(ierr);
Vec y; // H field vector if x_type == Etype
ierr = VecDuplicate(gi.vecTemp, &y); CHKERRQ(ierr);
ierr = VecCopy(conjSrc, y); CHKERRQ(ierr);
if (gi.x_type==Etype) {
ierr = MatMultAdd(CF, x, y, y); CHKERRQ(ierr);
ierr = VecScale(y, -1.0/PETSC_i/gi.omega); CHKERRQ(ierr);
} else {
ierr = VecScale(y, -1.0); CHKERRQ(ierr);
ierr = MatMultAdd(CF, x, y, y); CHKERRQ(ierr);
ierr = VecScale(y, 1.0/PETSC_i/gi.omega); CHKERRQ(ierr);
}
ierr = VecPointwiseDivide(y, y, conjParam);
PetscViewer viewer;
//viewer = PETSC_VIEWER_STDOUT_WORLD;
//ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD, h_file, FILE_MODE_WRITE, &viewer); CHKERRQ(ierr);
/** Write the E-field file. */
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD, e_file, FILE_MODE_WRITE, &viewer); CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer, "/");
if (gi.x_type==Etype) {
ierr = PetscObjectSetName((PetscObject) x, "E"); CHKERRQ(ierr);
ierr = VecView(x, viewer); CHKERRQ(ierr);
} else {
assert(gi.x_type==Htype);
ierr = PetscObjectSetName((PetscObject) y, "E"); CHKERRQ(ierr);
ierr = VecView(y, viewer); CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr);
/** Write the H-field file. */
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD, h_file, FILE_MODE_WRITE, &viewer); CHKERRQ(ierr);
ierr = PetscViewerHDF5PushGroup(viewer, "/");
if (gi.x_type==Etype) {
ierr = PetscObjectSetName((PetscObject) y, "H"); CHKERRQ(ierr);
ierr = VecView(y, viewer); CHKERRQ(ierr);
} else {
assert(gi.x_type==Htype);
ierr = PetscObjectSetName((PetscObject) x, "H"); CHKERRQ(ierr);
ierr = VecView(x, viewer); CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr);
ierr = VecDestroy(&y); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
DM dm; /* Problem specification */
DM dmAux; /* Material specification */
SNES snes; /* nonlinear solver */
Vec u,r; /* solution, residual vectors */
Mat A,J; /* Jacobian matrix */
MatNullSpace nullSpace; /* May be necessary for Neumann conditions */
AppCtx user; /* user-defined work context */
JacActionCtx userJ; /* context for Jacobian MF action */
PetscInt its; /* iterations for convergence */
PetscReal error = 0.0; /* L_2 error in the solution */
PetscInt numComponents;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL, help);CHKERRQ(ierr);
ierr = ProcessOptions(PETSC_COMM_WORLD, &user);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD, &snes);CHKERRQ(ierr);
ierr = CreateMesh(PETSC_COMM_WORLD, &user, &dm);CHKERRQ(ierr);
ierr = SNESSetDM(snes, dm);CHKERRQ(ierr);
ierr = DMSetApplicationContext(dm, &user);CHKERRQ(ierr);
user.fem.fe = user.fe;
ierr = PetscFECreateDefault(dm, user.dim, 1, PETSC_TRUE, NULL, -1, &user.fe[0]);CHKERRQ(ierr);
if (user.bcType != NEUMANN) {
user.fem.feBd = NULL; user.feBd[0] = NULL;
} else {
user.fem.feBd = user.feBd;
ierr = PetscFECreateDefault(dm, user.dim-1, 1, PETSC_TRUE, "bd_", -1, &user.feBd[0]);CHKERRQ(ierr);
}
ierr = PetscFEGetNumComponents(user.fe[0], &numComponents);CHKERRQ(ierr);
ierr = PetscMalloc(NUM_FIELDS * sizeof(void (*)(const PetscReal[], PetscScalar *, void *)), &user.exactFuncs);CHKERRQ(ierr);
ierr = SetupProblem(dm, &user);CHKERRQ(ierr);
ierr = SetupSection(dm, &user);CHKERRQ(ierr);
/* Setup Material */
ierr = DMClone(dm, &dmAux);CHKERRQ(ierr);
ierr = DMPlexCopyCoordinates(dm, dmAux);CHKERRQ(ierr);
if (user.variableCoefficient != COEFF_FIELD) {
user.fem.feAux = NULL; user.feAux[0] = NULL;
} else {
PetscQuadrature q;
user.fem.feAux = user.feAux;
ierr = PetscFECreateDefault(dmAux, user.dim, 1, PETSC_TRUE, "mat_", -1, &user.feAux[0]);CHKERRQ(ierr);
ierr = PetscFEGetQuadrature(user.fe[0], &q);CHKERRQ(ierr);
ierr = PetscFESetQuadrature(user.feAux[0], q);CHKERRQ(ierr);
}
ierr = SetupMaterialSection(dmAux, &user);CHKERRQ(ierr);
ierr = SetupMaterial(dm, dmAux, &user);CHKERRQ(ierr);
ierr = DMDestroy(&dmAux);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm, &u);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) u, "potential");CHKERRQ(ierr);
ierr = VecDuplicate(u, &r);CHKERRQ(ierr);
ierr = DMSetMatType(dm,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(dm, &J);CHKERRQ(ierr);
if (user.jacobianMF) {
PetscInt M, m, N, n;
ierr = MatGetSize(J, &M, &N);CHKERRQ(ierr);
ierr = MatGetLocalSize(J, &m, &n);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD, &A);CHKERRQ(ierr);
ierr = MatSetSizes(A, m, n, M, N);CHKERRQ(ierr);
ierr = MatSetType(A, MATSHELL);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
#if 0
ierr = MatShellSetOperation(A, MATOP_MULT, (void (*)(void))FormJacobianAction);CHKERRQ(ierr);
#endif
userJ.dm = dm;
userJ.J = J;
userJ.user = &user;
ierr = DMCreateLocalVector(dm, &userJ.u);CHKERRQ(ierr);
ierr = DMPlexProjectFunctionLocal(dm, user.fe, user.exactFuncs, NULL, INSERT_BC_VALUES, userJ.u);CHKERRQ(ierr);
ierr = MatShellSetContext(A, &userJ);CHKERRQ(ierr);
} else {
A = J;
}
if (user.bcType == NEUMANN) {
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject) dm), PETSC_TRUE, 0, NULL, &nullSpace);CHKERRQ(ierr);
ierr = MatSetNullSpace(J, nullSpace);CHKERRQ(ierr);
if (A != J) {
ierr = MatSetNullSpace(A, nullSpace);CHKERRQ(ierr);
}
}
ierr = DMSNESSetFunctionLocal(dm, (PetscErrorCode (*)(DM,Vec,Vec,void*)) DMPlexComputeResidualFEM, &user);CHKERRQ(ierr);
ierr = DMSNESSetJacobianLocal(dm, (PetscErrorCode (*)(DM,Vec,Mat,Mat,void*)) DMPlexComputeJacobianFEM, &user);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, A, J, NULL, NULL);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = DMPlexProjectFunction(dm, user.fe, user.exactFuncs, NULL, INSERT_ALL_VALUES, u);CHKERRQ(ierr);
if (user.checkpoint) {
#if defined(PETSC_HAVE_HDF5)
ierr = PetscViewerHDF5PushGroup(user.checkpoint, "/fields");CHKERRQ(ierr);
ierr = VecLoad(u, user.checkpoint);CHKERRQ(ierr);
ierr = PetscViewerHDF5PopGroup(user.checkpoint);CHKERRQ(ierr);
#endif
}
ierr = PetscViewerDestroy(&user.checkpoint);CHKERRQ(ierr);
if (user.showInitial) {
Vec lv;
ierr = DMGetLocalVector(dm, &lv);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, u, INSERT_VALUES, lv);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, u, INSERT_VALUES, lv);CHKERRQ(ierr);
ierr = DMPrintLocalVec(dm, "Local function", 1.0e-10, lv);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &lv);CHKERRQ(ierr);
}
if (user.runType == RUN_FULL) {
void (*initialGuess[numComponents])(const PetscReal x[], PetscScalar *, void *ctx);
PetscInt c;
for (c = 0; c < numComponents; ++c) initialGuess[c] = zero;
ierr = DMPlexProjectFunction(dm, user.fe, initialGuess, NULL, INSERT_VALUES, u);CHKERRQ(ierr);
if (user.debug) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial guess\n");CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = SNESSolve(snes, NULL, u);CHKERRQ(ierr);
ierr = SNESGetIterationNumber(snes, &its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Number of SNES iterations = %D\n", its);CHKERRQ(ierr);
ierr = DMPlexComputeL2Diff(dm, user.fe, user.exactFuncs, NULL, u, &error);CHKERRQ(ierr);
if (error < 1.0e-11) {ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: < 1.0e-11\n");CHKERRQ(ierr);}
else {ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %g\n", error);CHKERRQ(ierr);}
if (user.showSolution) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Solution\n");CHKERRQ(ierr);
ierr = VecChop(u, 3.0e-9);CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
} else if (user.runType == RUN_PERF) {
PetscReal res = 0.0;
ierr = SNESComputeFunction(snes, u, r);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial Residual\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Residual: %g\n", res);CHKERRQ(ierr);
} else {
PetscReal res = 0.0;
/* Check discretization error */
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial guess\n");CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMPlexComputeL2Diff(dm, user.fe, user.exactFuncs, NULL, u, &error);CHKERRQ(ierr);
if (error < 1.0e-11) {ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: < 1.0e-11\n");CHKERRQ(ierr);}
else {ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %g\n", error);CHKERRQ(ierr);}
/* Check residual */
ierr = SNESComputeFunction(snes, u, r);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial Residual\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecView(r, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Residual: %g\n", res);CHKERRQ(ierr);
/* Check Jacobian */
{
Vec b;
ierr = SNESComputeJacobian(snes, u, A, A);CHKERRQ(ierr);
ierr = VecDuplicate(u, &b);CHKERRQ(ierr);
ierr = VecSet(r, 0.0);CHKERRQ(ierr);
ierr = SNESComputeFunction(snes, r, b);CHKERRQ(ierr);
ierr = MatMult(A, u, r);CHKERRQ(ierr);
ierr = VecAXPY(r, 1.0, b);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Au - b = Au + F(0)\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecView(r, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Linear L_2 Residual: %g\n", res);CHKERRQ(ierr);
}
}
ierr = VecViewFromOptions(u, NULL, "-vec_view");CHKERRQ(ierr);
if (user.bcType == NEUMANN) {
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
}
if (user.jacobianMF) {
ierr = VecDestroy(&userJ.u);CHKERRQ(ierr);
}
if (A != J) {ierr = MatDestroy(&A);CHKERRQ(ierr);}
ierr = PetscFEDestroy(&user.fe[0]);CHKERRQ(ierr);
ierr = PetscFEDestroy(&user.feBd[0]);CHKERRQ(ierr);
ierr = PetscFEDestroy(&user.feAux[0]);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFree(user.exactFuncs);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatLoad_AIJ_HDF5(Mat mat, PetscViewer viewer)
{
PetscViewerFormat format;
const PetscInt *i_glob = NULL;
PetscInt *i = NULL;
const PetscInt *j = NULL;
const PetscScalar *a = NULL;
const char *a_name = NULL, *i_name = NULL, *j_name = NULL, *mat_name = NULL, *c_name = NULL;
PetscInt p, m, M, N;
PetscInt bs = mat->rmap->bs;
PetscBool flg;
IS is_i = NULL, is_j = NULL;
Vec vec_a = NULL;
PetscLayout jmap = NULL;
MPI_Comm comm;
PetscMPIInt rank, size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
switch (format) {
case PETSC_VIEWER_HDF5_PETSC:
case PETSC_VIEWER_DEFAULT:
case PETSC_VIEWER_NATIVE:
case PETSC_VIEWER_HDF5_MAT:
break;
default:
SETERRQ1(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"PetscViewerFormat %s not supported for HDF5 input.",PetscViewerFormats[format]);
}
ierr = PetscObjectGetComm((PetscObject)mat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)mat,&mat_name);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetAIJNames(viewer,&i_name,&j_name,&a_name,&c_name);CHKERRQ(ierr);
ierr = PetscOptionsBegin(comm,NULL,"Options for loading matrix from HDF5","Mat");CHKERRQ(ierr);
ierr = PetscOptionsInt("-matload_block_size","Set the blocksize used to store the matrix","MatLoad",bs,&bs,&flg);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
if (flg) {
ierr = MatSetBlockSize(mat, bs);CHKERRQ(ierr);
}
ierr = PetscViewerHDF5PushGroup(viewer,mat_name);CHKERRQ(ierr);
ierr = PetscViewerHDF5ReadAttribute(viewer,NULL,c_name,PETSC_INT,&N);CHKERRQ(ierr);
ierr = PetscViewerHDF5ReadSizes(viewer, i_name, NULL, &M);CHKERRQ(ierr);
--M; /* i has size M+1 as there is global number of nonzeros stored at the end */
if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) {
/* Swap row and columns layout for unallocated matrix. I want to avoid calling MatTranspose() just to transpose sparsity pattern and layout. */
if (!mat->preallocated) {
PetscLayout tmp;
tmp = mat->rmap; mat->rmap = mat->cmap; mat->cmap = tmp;
} else SETERRQ(comm,PETSC_ERR_SUP,"Not for preallocated matrix - we would need to transpose it here which we want to avoid");
}
/* If global sizes are set, check if they are consistent with that given in the file */
if (mat->rmap->N >= 0 && mat->rmap->N != M) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of rows: Matrix in file has (%D) and input matrix has (%D)",mat->rmap->N,M);
if (mat->cmap->N >= 0 && mat->cmap->N != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of cols: Matrix in file has (%D) and input matrix has (%D)",mat->cmap->N,N);
/* Determine ownership of all (block) rows and columns */
mat->rmap->N = M;
mat->cmap->N = N;
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
m = mat->rmap->n;
/* Read array i (array of row indices) */
ierr = PetscMalloc1(m+1, &i);CHKERRQ(ierr); /* allocate i with one more position for local number of nonzeros on each rank */
i[0] = i[m] = 0; /* make the last entry always defined - the code block below overwrites it just on last rank */
if (rank == size-1) m++; /* in the loaded array i_glob, only the last rank has one more position with the global number of nonzeros */
M++;
ierr = ISCreate(comm,&is_i);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)is_i,i_name);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(is_i->map,m);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(is_i->map,M);CHKERRQ(ierr);
ierr = ISLoad(is_i,viewer);CHKERRQ(ierr);
ierr = ISGetIndices(is_i,&i_glob);CHKERRQ(ierr);
ierr = PetscMemcpy(i,i_glob,m*sizeof(PetscInt));CHKERRQ(ierr);
/* Reset m and M to the matrix sizes */
m = mat->rmap->n;
M--;
/* Create PetscLayout for j and a vectors; construct ranges first */
ierr = PetscLayoutCreate(comm,&jmap);CHKERRQ(ierr);
ierr = PetscCalloc1(size+1, &jmap->range);CHKERRQ(ierr);
ierr = MPI_Allgather(&i[0], 1, MPIU_INT, jmap->range, 1, MPIU_INT, comm);CHKERRQ(ierr);
/* Last rank has global number of nonzeros (= length of j and a arrays) in i[m] (last i entry) so broadcast it */
jmap->range[size] = i[m];
ierr = MPI_Bcast(&jmap->range[size], 1, MPIU_INT, size-1, comm);CHKERRQ(ierr);
for (p=size-1; p>0; p--) {
if (!jmap->range[p]) jmap->range[p] = jmap->range[p+1]; /* for ranks with 0 rows, take the value from the next processor */
}
i[m] = jmap->range[rank+1]; /* i[m] (last i entry) is equal to next rank's offset */
/* Deduce rstart, rend, n and N from the ranges */
ierr = PetscLayoutSetUp(jmap);CHKERRQ(ierr);
/* Convert global to local indexing of rows */
for (p=1; p<m+1; ++p) i[p] -= i[0];
i[0] = 0;
/* Read array j (array of column indices) */
ierr = ISCreate(comm,&is_j);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)is_j,j_name);CHKERRQ(ierr);
ierr = PetscLayoutDuplicate(jmap,&is_j->map);CHKERRQ(ierr);
ierr = ISLoad(is_j,viewer);CHKERRQ(ierr);
ierr = ISGetIndices(is_j,&j);CHKERRQ(ierr);
/* Read array a (array of values) */
ierr = VecCreate(comm,&vec_a);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)vec_a,a_name);CHKERRQ(ierr);
ierr = PetscLayoutDuplicate(jmap,&vec_a->map);CHKERRQ(ierr);
ierr = VecLoad(vec_a,viewer);CHKERRQ(ierr);
ierr = VecGetArrayRead(vec_a,&a);CHKERRQ(ierr);
/* populate matrix */
if (!((PetscObject)mat)->type_name) {
ierr = MatSetType(mat,MATAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr);
/*
ierr = MatSeqBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr);
*/
if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) {
/* Transpose the input matrix back */
ierr = MatTranspose(mat,MAT_INPLACE_MATRIX,&mat);CHKERRQ(ierr);
}
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&jmap);CHKERRQ(ierr);
ierr = PetscFree(i);CHKERRQ(ierr);
ierr = ISRestoreIndices(is_i,&i_glob);CHKERRQ(ierr);
ierr = ISRestoreIndices(is_j,&j);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(vec_a,&a);CHKERRQ(ierr);
ierr = ISDestroy(&is_i);CHKERRQ(ierr);
ierr = ISDestroy(&is_j);CHKERRQ(ierr);
ierr = VecDestroy(&vec_a);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
