PetscErrorCode NavierStokesSolver::stepTime()
{
PetscErrorCode ierr;
//solve intermediate velocity A q_star = r1
ierr = PetscLogStagePush(stageSolveIntermediateVelocity); CHKERRQ(ierr);
ierr = calculateExplicitTerms(); CHKERRQ(ierr);
ierr = updateBoundaryGhosts(); CHKERRQ(ierr);
ierr = generateBC1(); CHKERRQ(ierr);
ierr = generateRHS1();CHKERRQ(ierr);
ierr = solveIntermediateVelocity();CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr);
//solve Poisson for pressure
ierr = PetscLogStagePush(stageSolvePoissonSystem); CHKERRQ(ierr);
ierr = generateBC2(); CHKERRQ(ierr);
ierr = generateRHS2(); CHKERRQ(ierr);
ierr = solvePoissonSystem(); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr);
//project pressure to satisfy continuity
ierr = PetscLogStagePush(stageProjectionStep); CHKERRQ(ierr);
ierr = projectionStep(); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr);
timeStep++;
return 0;
}
static PetscErrorCode TestCone(DM dm, AppCtx *user)
{
PetscInt numRuns, cStart, cEnd, c, i;
PetscReal maxTimePerRun = user->maxConeTime;
PetscLogStage stage;
PetscLogEvent event;
PetscEventPerfInfo eventInfo;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogStageRegister("DMPlex Cone Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("Cone", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr);
for (i = 0; i < user->iterations; ++i) {
for (c = cStart; c < cEnd; ++c) {
const PetscInt *cone;
ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
}
}
ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
numRuns = (cEnd-cStart) * user->iterations;
if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1);
if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0);
if (eventInfo.time > maxTimePerRun * numRuns) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Cones: %d Average time per cone: %gs standard: %gs\n", numRuns, eventInfo.time/numRuns, maxTimePerRun);
if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for cone %g > standard %g", eventInfo.time/numRuns, maxTimePerRun);
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCApply_MG(PC pc,Vec b,Vec x)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
PetscInt levels = mglevels[0]->levels,i;
PetscFunctionBegin;
if (mg->stageApply) {ierr = PetscLogStagePush(mg->stageApply);CHKERRQ(ierr);}
/* When the DM is supplying the matrix then it will not exist until here */
for (i=0; i<levels; i++) {
if (!mglevels[i]->A) {
ierr = KSPGetOperators(mglevels[i]->smoothu,&mglevels[i]->A,NULL);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)mglevels[i]->A);CHKERRQ(ierr);
}
}
mglevels[levels-1]->b = b;
mglevels[levels-1]->x = x;
if (mg->am == PC_MG_MULTIPLICATIVE) {
ierr = VecSet(x,0.0);CHKERRQ(ierr);
for (i=0; i<mg->cyclesperpcapply; i++) {
ierr = PCMGMCycle_Private(pc,mglevels+levels-1,NULL);CHKERRQ(ierr);
}
} else if (mg->am == PC_MG_ADDITIVE) {
ierr = PCMGACycle_Private(pc,mglevels);CHKERRQ(ierr);
} else if (mg->am == PC_MG_KASKADE) {
ierr = PCMGKCycle_Private(pc,mglevels);CHKERRQ(ierr);
} else {
ierr = PCMGFCycle_Private(pc,mglevels);CHKERRQ(ierr);
}
if (mg->stageApply) {ierr = PetscLogStagePop();CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
PetscErrorCode NavierStokesSolver::initialize()
{
PetscErrorCode ierr;
ierr = PetscLogStagePush(stageInitialize); CHKERRQ(ierr);
initialfluid();
ierr = createDMs(); CHKERRQ(ierr);
ierr = initializeCommon(); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr);
return 0;
}
PetscErrorCode TestVecClosure(DM dm, AppCtx *user)
{
PetscSection s;
Vec v;
PetscInt numRuns, cStart, cEnd, c, i;
PetscScalar tmpArray[64];
PetscScalar *userArray = user->reuseArray ? tmpArray : NULL;
PetscReal maxTimePerRun = user->maxVecClosureTime;
PetscStageLog stageLog;
PetscEventPerfLog eventLog;
PetscInt stage;
PetscLogEvent event;
PetscEventPerfInfo eventInfo;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogStageRegister("DMPlex Vector Closure Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("VecClosure", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = DMPlexCreateSection(dm, user->dim, user->numFields, user->numComponents, user->numDof, 0, NULL, NULL, &s);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dm, s);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &v);CHKERRQ(ierr);
ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr);
for (i = 0; i < user->iterations; ++i) {
for (c = cStart; c < cEnd; ++c) {
PetscScalar *closure = userArray;
PetscInt closureSize = 64;;
ierr = DMPlexVecGetClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr);
if (!user->reuseArray) {ierr = DMPlexVecRestoreClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr);}
}
}
ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &v);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogGetStageLog(&stageLog);
ierr = PetscStageLogGetEventPerfLog(stageLog, stage, &eventLog);
numRuns = (cEnd-cStart) * user->iterations;
eventInfo = eventLog->eventInfo[event];
if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1);
if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0);
if (eventInfo.time > maxTimePerRun * numRuns) {
ierr = PetscPrintf(PETSC_COMM_SELF, "VecClosures: %d Average time per cone: %gs standard: %gs\n", numRuns, eventInfo.time/numRuns, maxTimePerRun);
if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for vector closure %g > standard %g", eventInfo.time/numRuns, maxTimePerRun);
}
PetscFunctionReturn(0);
}
// assemble the right-hand side of the system for the Lagrangian forces
PetscErrorCode RigidKinematicsSolver::assembleRHSForces()
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscLogStagePush(stageRHSForces); CHKERRQ(ierr);
// rhsf = UB - E u^{**}
ierr = MatMult(E, solution->UGlobal, rhsf); CHKERRQ(ierr);
ierr = VecScale(rhsf, -1.0); CHKERRQ(ierr);
ierr = VecAYPX(rhsf, 1.0, UB); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr);
PetscFunctionReturn(0);
} // assembleRHSForces
// write the solution, solver info, and body points to files
PetscErrorCode RigidKinematicsSolver::write()
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DecoupledIBPMSolver::write(); CHKERRQ(ierr);
if (ite % nsave == 0)
{
ierr = PetscLogStagePush(stageWrite); CHKERRQ(ierr);
ierr = writeBodies(); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr); // end of stageWrite
}
PetscFunctionReturn(0);
} // write
// initialize the decoupled IBPM solver for moving rigid bodies
PetscErrorCode RigidKinematicsSolver::init(const MPI_Comm &world,
const YAML::Node &node)
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DecoupledIBPMSolver::init(world, node); CHKERRQ(ierr);
ierr = PetscLogStagePush(stageInitialize); CHKERRQ(ierr);
ierr = PetscLogStageRegister("moveIB", &stageMoveIB); CHKERRQ(ierr);
ierr = VecDuplicate(f, &UB); CHKERRQ(ierr);
ierr = VecSet(UB, 0.0); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr); // end of stageInitialize
PetscFunctionReturn(0);
} // init
PetscErrorCode TestTransitiveClosure(DM dm, AppCtx *user)
{
PetscInt numRuns, cStart, cEnd, c, i;
PetscReal maxTimePerRun = user->maxClosureTime;
PetscStageLog stageLog;
PetscEventPerfLog eventLog;
PetscInt stage;
PetscLogEvent event;
PetscEventPerfInfo eventInfo;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogStageRegister("DMPlex Transitive Closure Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("TransitiveClosure", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr);
for (i = 0; i < user->iterations; ++i) {
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL;
PetscInt closureSize;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
}
ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogGetStageLog(&stageLog);
ierr = PetscStageLogGetEventPerfLog(stageLog, stage, &eventLog);
numRuns = (cEnd-cStart) * user->iterations;
eventInfo = eventLog->eventInfo[event];
if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1);
if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0);
if (eventInfo.time > maxTimePerRun * numRuns) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Closures: %d Average time per cone: %gs standard: %gs\n", numRuns, eventInfo.time/numRuns, maxTimePerRun);
if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for closure %g > standard %g", eventInfo.time/numRuns, maxTimePerRun);
}
PetscFunctionReturn(0);
}
// update Lagrangian points, boundary velocity, and operators
PetscErrorCode RigidKinematicsSolver::moveBodies(const PetscReal &ti)
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscLogStagePush(stageMoveIB); CHKERRQ(ierr);
ierr = setCoordinatesBodies(ti); CHKERRQ(ierr);
ierr = setVelocityBodies(ti); CHKERRQ(ierr);
ierr = bodies->updateMeshIdx(mesh); CHKERRQ(ierr);
if (E != PETSC_NULL) {ierr = MatDestroy(&E); CHKERRQ(ierr);}
if (H != PETSC_NULL) {ierr = MatDestroy(&H); CHKERRQ(ierr);}
if (BNH != PETSC_NULL) {ierr = MatDestroy(&BNH); CHKERRQ(ierr);}
if (EBNH != PETSC_NULL) {ierr = MatDestroy(&EBNH); CHKERRQ(ierr);}
ierr = createExtraOperators(); CHKERRQ(ierr);
ierr = fSolver->setMatrix(EBNH); CHKERRQ(ierr);
ierr = PetscLogStagePop(); CHKERRQ(ierr); // end of stageMoveIB
PetscFunctionReturn(0);
} // moveBodies
int main(int argc,char **args)
{
Mat Amat;
PetscErrorCode ierr;
SNES snes;
KSP ksp;
MPI_Comm comm;
PetscMPIInt npe,rank;
PetscLogStage stage[7];
PetscBool test_nonzero_cols=PETSC_FALSE,use_nearnullspace=PETSC_TRUE;
Vec xx,bb;
PetscInt iter,i,N,dim=3,cells[3]={1,1,1},max_conv_its,local_sizes[7],run_type=1;
DM dm,distdm,basedm;
PetscBool flg;
char convType[256];
PetscReal Lx,mdisp[10],err[10];
const char * const options[10] = {"-ex56_dm_refine 0",
"-ex56_dm_refine 1",
"-ex56_dm_refine 2",
"-ex56_dm_refine 3",
"-ex56_dm_refine 4",
"-ex56_dm_refine 5",
"-ex56_dm_refine 6",
"-ex56_dm_refine 7",
"-ex56_dm_refine 8",
"-ex56_dm_refine 9"};
PetscFunctionBeginUser;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
comm = PETSC_COMM_WORLD;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &npe);CHKERRQ(ierr);
/* options */
ierr = PetscOptionsBegin(comm,NULL,"3D bilinear Q1 elasticity options","");CHKERRQ(ierr);
{
i = 3;
ierr = PetscOptionsIntArray("-cells", "Number of (flux tube) processor in each dimension", "ex56.c", cells, &i, NULL);CHKERRQ(ierr);
Lx = 1.; /* or ne for rod */
max_conv_its = 3;
ierr = PetscOptionsInt("-max_conv_its","Number of iterations in convergence study","",max_conv_its,&max_conv_its,NULL);CHKERRQ(ierr);
if (max_conv_its<=0 || max_conv_its>7) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_USER, "Bad number of iterations for convergence test (%D)",max_conv_its);
ierr = PetscOptionsReal("-lx","Length of domain","",Lx,&Lx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-alpha","material coefficient inside circle","",s_soft_alpha,&s_soft_alpha,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-test_nonzero_cols","nonzero test","",test_nonzero_cols,&test_nonzero_cols,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-use_mat_nearnullspace","MatNearNullSpace API test","",use_nearnullspace,&use_nearnullspace,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-run_type","0: twisting load on cantalever, 1: 3rd order accurate convergence test","",run_type,&run_type,NULL);CHKERRQ(ierr);
i = 3;
ierr = PetscOptionsInt("-mat_block_size","","",i,&i,&flg);CHKERRQ(ierr);
if (!flg || i!=3) SETERRQ2(PETSC_COMM_WORLD, PETSC_ERR_USER, "'-mat_block_size 3' must be set (%D) and = 3 (%D)",flg,flg? i : 3);
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Mesh Setup", &stage[6]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("1st Setup", &stage[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("1st Solve", &stage[1]);CHKERRQ(ierr);
/* create DM, Plex calls DMSetup */
ierr = PetscLogStagePush(stage[6]);CHKERRQ(ierr);
ierr = DMPlexCreateHexBoxMesh(comm, dim, cells, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, &dm);CHKERRQ(ierr);
{
DMLabel label;
IS is;
ierr = DMCreateLabel(dm, "boundary");CHKERRQ(ierr);
ierr = DMGetLabel(dm, "boundary", &label);CHKERRQ(ierr);
ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
if (run_type==0) {
ierr = DMGetStratumIS(dm, "boundary", 1, &is);CHKERRQ(ierr);
ierr = DMCreateLabel(dm,"Faces");CHKERRQ(ierr);
if (is) {
PetscInt d, f, Nf;
const PetscInt *faces;
PetscInt csize;
PetscSection cs;
Vec coordinates ;
DM cdm;
ierr = ISGetLocalSize(is, &Nf);CHKERRQ(ierr);
ierr = ISGetIndices(is, &faces);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
ierr = DMGetDefaultSection(cdm, &cs);CHKERRQ(ierr);
/* Check for each boundary face if any component of its centroid is either 0.0 or 1.0 */
for (f = 0; f < Nf; ++f) {
PetscReal faceCoord;
PetscInt b,v;
PetscScalar *coords = NULL;
PetscInt Nv;
ierr = DMPlexVecGetClosure(cdm, cs, coordinates, faces[f], &csize, &coords);CHKERRQ(ierr);
Nv = csize/dim; /* Calculate mean coordinate vector */
for (d = 0; d < dim; ++d) {
faceCoord = 0.0;
for (v = 0; v < Nv; ++v) faceCoord += PetscRealPart(coords[v*dim+d]);
faceCoord /= Nv;
for (b = 0; b < 2; ++b) {
if (PetscAbs(faceCoord - b) < PETSC_SMALL) { /* domain have not been set yet, still [0,1]^3 */
ierr = DMSetLabelValue(dm, "Faces", faces[f], d*2+b+1);CHKERRQ(ierr);
}
}
}
ierr = DMPlexVecRestoreClosure(cdm, cs, coordinates, faces[f], &csize, &coords);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(is, &faces);CHKERRQ(ierr);
}
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = DMGetLabel(dm, "Faces", &label);CHKERRQ(ierr);
ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
}
}
{
PetscInt dimEmbed, i;
PetscInt nCoords;
PetscScalar *coords,bounds[] = {0,Lx,-.5,.5,-.5,.5,}; /* x_min,x_max,y_min,y_max */
Vec coordinates;
if (run_type==1) {
for (i = 0; i < 2*dim; i++) bounds[i] = (i%2) ? 1 : 0;
}
ierr = DMGetCoordinatesLocal(dm,&coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dm,&dimEmbed);CHKERRQ(ierr);
if (dimEmbed != dim) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"dimEmbed != dim %D",dimEmbed);CHKERRQ(ierr);
ierr = VecGetLocalSize(coordinates,&nCoords);CHKERRQ(ierr);
if (nCoords % dimEmbed) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Coordinate vector the wrong size");CHKERRQ(ierr);
ierr = VecGetArray(coordinates,&coords);CHKERRQ(ierr);
for (i = 0; i < nCoords; i += dimEmbed) {
PetscInt j;
PetscScalar *coord = &coords[i];
for (j = 0; j < dimEmbed; j++) {
coord[j] = bounds[2 * j] + coord[j] * (bounds[2 * j + 1] - bounds[2 * j]);
}
}
ierr = VecRestoreArray(coordinates,&coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dm,coordinates);CHKERRQ(ierr);
}
/* convert to p4est, and distribute */
ierr = PetscOptionsBegin(comm, "", "Mesh conversion options", "DMPLEX");CHKERRQ(ierr);
ierr = PetscOptionsFList("-dm_type","Convert DMPlex to another format (should not be Plex!)","ex56.c",DMList,DMPLEX,convType,256,&flg);CHKERRQ(ierr);
ierr = PetscOptionsEnd();
if (flg) {
DM newdm;
ierr = DMConvert(dm,convType,&newdm);CHKERRQ(ierr);
if (newdm) {
const char *prefix;
PetscBool isForest;
ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject)newdm,prefix);CHKERRQ(ierr);
ierr = DMIsForest(newdm,&isForest);CHKERRQ(ierr);
if (isForest) {
} else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Converted to non Forest?");
ierr = DMDestroy(&dm);CHKERRQ(ierr);
dm = newdm;
} else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Convert failed?");
} else {
/* Plex Distribute mesh over processes */
ierr = DMPlexDistribute(dm, 0, NULL, &distdm);CHKERRQ(ierr);
if (distdm) {
const char *prefix;
ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject)distdm,prefix);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
dm = distdm;
}
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
basedm = dm; dm = NULL;
for (iter=0 ; iter<max_conv_its ; iter++) {
ierr = PetscLogStagePush(stage[6]);CHKERRQ(ierr);
/* make new DM */
ierr = DMClone(basedm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, "ex56_");CHKERRQ(ierr);
ierr = PetscObjectSetName( (PetscObject)dm,"Mesh");CHKERRQ(ierr);
ierr = PetscOptionsClearValue(NULL,"-ex56_dm_refine");CHKERRQ(ierr);
ierr = PetscOptionsInsertString(NULL,options[iter]);CHKERRQ(ierr);
ierr = DMSetFromOptions(dm);CHKERRQ(ierr); /* refinement done here in Plex, p4est */
/* snes */
ierr = SNESCreate(comm, &snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes, dm);CHKERRQ(ierr);
/* fem */
{
const PetscInt Ncomp = dim;
const PetscInt components[] = {0,1,2};
const PetscInt Nfid = 1, Npid = 1;
const PetscInt fid[] = {1}; /* The fixed faces (x=0) */
const PetscInt pid[] = {2}; /* The faces with loading (x=L_x) */
PetscFE fe;
PetscDS prob;
DM cdm = dm;
ierr = PetscFECreateDefault(dm, dim, dim, PETSC_FALSE, NULL, PETSC_DECIDE, &fe);CHKERRQ(ierr); /* elasticity */
ierr = PetscObjectSetName((PetscObject) fe, "deformation");CHKERRQ(ierr);
/* FEM prob */
ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
ierr = PetscDSSetDiscretization(prob, 0, (PetscObject) fe);CHKERRQ(ierr);
/* setup problem */
if (run_type==1) {
ierr = PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d);CHKERRQ(ierr);
ierr = PetscDSSetResidual(prob, 0, f0_u_x4, f1_u_3d);CHKERRQ(ierr);
} else {
ierr = PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d_alpha);CHKERRQ(ierr);
ierr = PetscDSSetResidual(prob, 0, f0_u, f1_u_3d_alpha);CHKERRQ(ierr);
ierr = PetscDSSetBdResidual(prob, 0, f0_bd_u_3d, f1_bd_u);CHKERRQ(ierr);
}
/* bcs */
if (run_type==1) {
PetscInt id = 1;
ierr = DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", "boundary", 0, 0, NULL, (void (*)()) zero, 1, &id, NULL);CHKERRQ(ierr);
} else {
ierr = PetscDSAddBoundary(prob, DM_BC_ESSENTIAL, "fixed", "Faces", 0, Ncomp, components, (void (*)()) zero, Nfid, fid, NULL);CHKERRQ(ierr);
ierr = PetscDSAddBoundary(prob, DM_BC_NATURAL, "traction", "Faces", 0, Ncomp, components, NULL, Npid, pid, NULL);CHKERRQ(ierr);
}
while (cdm) {
ierr = DMSetDS(cdm,prob);CHKERRQ(ierr);
ierr = DMGetCoarseDM(cdm, &cdm);CHKERRQ(ierr);
}
ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
}
/* vecs & mat */
ierr = DMCreateGlobalVector(dm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx, &bb);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) bb, "b");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) xx, "u");CHKERRQ(ierr);
ierr = DMCreateMatrix(dm, &Amat);CHKERRQ(ierr);
ierr = VecGetSize(bb,&N);CHKERRQ(ierr);
local_sizes[iter] = N;
ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %d global equations, %d vertices\n",rank,PETSC_FUNCTION_NAME,N,N/dim);CHKERRQ(ierr);
if (use_nearnullspace && N/dim > 1) {
/* Set up the near null space (a.k.a. rigid body modes) that will be used by the multigrid preconditioner */
DM subdm;
MatNullSpace nearNullSpace;
PetscInt fields = 0;
PetscObject deformation;
ierr = DMCreateSubDM(dm, 1, &fields, NULL, &subdm);CHKERRQ(ierr);
ierr = DMPlexCreateRigidBody(subdm, &nearNullSpace);CHKERRQ(ierr);
ierr = DMGetField(dm, 0, &deformation);CHKERRQ(ierr);
ierr = PetscObjectCompose(deformation, "nearnullspace", (PetscObject) nearNullSpace);CHKERRQ(ierr);
ierr = DMDestroy(&subdm);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nearNullSpace);CHKERRQ(ierr); /* created by DM and destroyed by Mat */
}
ierr = DMPlexSetSNESLocalFEM(dm,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, Amat, Amat, NULL, NULL);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
/* ksp */
ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr);
/* test BCs */
ierr = VecZeroEntries(xx);CHKERRQ(ierr);
if (test_nonzero_cols) {
if (rank==0) ierr = VecSetValue(xx,0,1.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(xx);CHKERRQ(ierr);
ierr = VecAssemblyEnd(xx);CHKERRQ(ierr);
}
ierr = VecZeroEntries(bb);CHKERRQ(ierr);
ierr = VecGetSize(bb,&i);CHKERRQ(ierr);
local_sizes[iter] = i;
ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %d equations in vector, %d vertices\n",rank,PETSC_FUNCTION_NAME,i,i/dim);CHKERRQ(ierr);
/* setup solver, dummy solve to really setup */
if (0) {
ierr = KSPSetTolerances(ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1);CHKERRQ(ierr);
ierr = SNESSolve(snes, bb, xx);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,50);CHKERRQ(ierr);
ierr = VecZeroEntries(xx);CHKERRQ(ierr);
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
/* solve */
ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr);
ierr = SNESSolve(snes, bb, xx);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = VecNorm(xx,NORM_INFINITY,&mdisp[iter]);CHKERRQ(ierr);
ierr = DMViewFromOptions(dm, NULL, "-dm_view");CHKERRQ(ierr);
{
PetscViewer viewer = NULL;
PetscViewerFormat fmt;
ierr = PetscOptionsGetViewer(comm,"ex56_","-vec_view",&viewer,&fmt,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerPushFormat(viewer,fmt);CHKERRQ(ierr);
ierr = VecView(xx,viewer);CHKERRQ(ierr);
ierr = VecView(bb,viewer);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
/* Free work space */
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&bb);CHKERRQ(ierr);
ierr = MatDestroy(&Amat);CHKERRQ(ierr);
}
ierr = DMDestroy(&basedm);CHKERRQ(ierr);
if (run_type==1) {
err[0] = 59.975208 - mdisp[0]; /* error with what I think is the exact solution */
} else {
err[0] = 171.038 - mdisp[0];
}
for (iter=1 ; iter<max_conv_its ; iter++) {
if (run_type==1) {
err[iter] = 59.975208 - mdisp[iter];
} else {
err[iter] = 171.038 - mdisp[iter];
}
PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %D) N=%12D, max displ=%9.7e, disp diff=%9.2e, error=%4.3e, rate=%3.2g\n",
rank,PETSC_FUNCTION_NAME,iter,local_sizes[iter],mdisp[iter],
mdisp[iter]-mdisp[iter-1],err[iter],log(err[iter-1]/err[iter])/log(2.));
}
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char ** argv)
{
PetscErrorCode ierr;
PetscInt i, nbranch = 0, eStart, eEnd, vStart, vEnd;
PetscInt seed = 0, nnode = 0;
PetscMPIInt size, rank;
DM networkdm;
Vec x, b;
Mat A;
KSP ksp;
PetscInt *edgelist = NULL;
PetscInt componentkey[2];
Node *node;
Branch *branch;
#if defined(PETSC_USE_LOG)
PetscLogStage stage[3];
#endif
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-seed",&seed,NULL);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Network Creation", &stage[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("DMNetwork data structures", &stage[1]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("KSP", &stage[2]);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
/* "read" data only for processor 0 */
if (!rank) {
nnode = 100;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&nnode,NULL);CHKERRQ(ierr);
ierr = random_network(nnode, &nbranch, &node, &branch, &edgelist, seed);CHKERRQ(ierr);
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr);
ierr = DMNetworkCreate(PETSC_COMM_WORLD,&networkdm);CHKERRQ(ierr);
ierr = DMNetworkRegisterComponent(networkdm,"nstr",sizeof(Node),&componentkey[0]);CHKERRQ(ierr);
ierr = DMNetworkRegisterComponent(networkdm,"bsrt",sizeof(Branch),&componentkey[1]);CHKERRQ(ierr);
/* Set number of nodes/edges */
ierr = DMNetworkSetSizes(networkdm,1,0,&nnode,&nbranch,NULL,NULL);CHKERRQ(ierr);
/* Add edge connectivity */
ierr = DMNetworkSetEdgeList(networkdm,&edgelist,NULL);CHKERRQ(ierr);
/* Set up the network layout */
ierr = DMNetworkLayoutSetUp(networkdm);CHKERRQ(ierr);
/* Add network components: physical parameters of nodes and branches*/
if (!rank) {
ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr);
for (i = eStart; i < eEnd; i++) {
ierr = DMNetworkAddComponent(networkdm,i,componentkey[1],&branch[i-eStart]);CHKERRQ(ierr);
ierr = DMNetworkAddNumVariables(networkdm,i,1);CHKERRQ(ierr);
}
ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr);
for (i = vStart; i < vEnd; i++) {
ierr = DMNetworkAddComponent(networkdm,i,componentkey[0],&node[i-vStart]);CHKERRQ(ierr);
/* Add number of variables */
ierr = DMNetworkAddNumVariables(networkdm,i,1);CHKERRQ(ierr);
}
}
/* Network partitioning and distribution of data */
ierr = DMSetUp(networkdm);CHKERRQ(ierr);
ierr = DMNetworkDistribute(&networkdm,0);CHKERRQ(ierr);
ierr = DMNetworkAssembleGraphStructures(networkdm);CHKERRQ(ierr);
/* We don't use these data structures anymore since they have been copied to networkdm */
if (!rank) {
ierr = PetscFree(edgelist);CHKERRQ(ierr);
ierr = PetscFree2(node,branch);CHKERRQ(ierr);
}
/* Create vectors and matrix */
ierr = DMCreateGlobalVector(networkdm,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = DMCreateMatrix(networkdm,&A);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stage[2]);CHKERRQ(ierr);
/* Assembly system of equations */
ierr = FormOperator(networkdm,A,b);CHKERRQ(ierr);
/* Solve linear system: A x = b */
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSolve(ksp, b, x);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
/* Free work space */
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = DMDestroy(&networkdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc, char ** argv ) {
int size, rank;
bool incCorner = 1;
unsigned int dim=3;
unsigned int maxDepth=29;
bool compressLut=true;
std::vector<ot::TreeNode> balOct;
double mgLoadFac = 2.0;
unsigned int regLev = 2;
PetscInitialize(&argc, &argv, "options", help);
ot::RegisterEvents();
ot::DAMG_Initialize(MPI_COMM_WORLD);
#ifdef PETSC_USE_LOG
PetscClassId classid;
PetscClassIdRegister("Dendro",&classid);
PetscLogEventRegister("matProp",classid, &matPropEvent);
PetscLogEventRegister("ODAmatDiag",classid, &Jac1DiagEvent);
PetscLogEventRegister("ODAmatMult",classid, &Jac1MultEvent);
PetscLogEventRegister("ODAmatDiagFinest",classid, &Jac1FinestDiagEvent);
PetscLogEventRegister("ODAmatMultFinest",classid, &Jac1FinestMultEvent);
PetscLogEventRegister("OMGmatDiag-2",classid, &Jac2DiagEvent);
PetscLogEventRegister("OMGmatMult-2",classid, &Jac2MultEvent);
PetscLogEventRegister("OMGmatDiagFinest-2",classid, &Jac2FinestDiagEvent);
PetscLogEventRegister("OMGmatMultFinest-2",classid, &Jac2FinestMultEvent);
PetscLogEventRegister("OMGmatDiag-3",classid, &Jac3DiagEvent);
PetscLogEventRegister("OMGmatMult-3",classid, &Jac3MultEvent);
PetscLogEventRegister("OMGmatDiagFinest-3",classid, &Jac3FinestDiagEvent);
PetscLogEventRegister("OMGmatMultFinest-3",classid, &Jac3FinestMultEvent);
int stages[1];
PetscLogStageRegister("Solve",&stages[0]);
#endif
MPI_Comm_size(MPI_COMM_WORLD,&size);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
if(argc > 1) {
regLev = atoi(argv[1]);
}
if(argc > 2) {
maxDepth = atoi(argv[2]);
}
if(argc > 3) {
dim = atoi(argv[3]);
}
if(argc > 4) { incCorner = (bool)(atoi(argv[4]));}
if(argc > 5) { compressLut = (bool)(atoi(argv[5]));}
if(argc > 6) { mgLoadFac = atof(argv[6]); }
#ifdef PETSC_USE_LOG
PetscLogStagePush(stages[0]);
#endif
MPI_Barrier(MPI_COMM_WORLD);
ot::DAMG *damg;
int nlevels = 1; //number of multigrid levels
unsigned int dof = 1; // degrees of freedom per node
createRegularOctree(balOct, regLev, dim, maxDepth, MPI_COMM_WORLD);
PetscInt nlevelsPetscInt = nlevels; //To keep the compilers happy when using 64-bit indices
PetscOptionsGetInt(0, "-nlevels", &nlevelsPetscInt, 0);
nlevels = nlevelsPetscInt;
// Note: The user context for all levels will be set separately later.
MPI_Barrier(MPI_COMM_WORLD);
if(!rank) {
std::cout<<" nlevels initial: "<<nlevels<<std::endl;
}
ot::DAMGCreateAndSetDA(PETSC_COMM_WORLD, nlevels, NULL, &damg,
balOct, dof, mgLoadFac, compressLut, incCorner);
if(!rank) {
std::cout<<" nlevels final: "<<nlevels<<std::endl;
}
MPI_Barrier(MPI_COMM_WORLD);
if(!rank) {
std::cout << "Created DA for all levels."<< std::endl;
}
MPI_Barrier(MPI_COMM_WORLD);
ot::PrintDAMG(damg);
MPI_Barrier(MPI_COMM_WORLD);
for(int i=0;i<nlevels;i++) {
bool isRegOct = isRegularGrid(damg[i]->da);
if(!rank) {
std::cout<<"Level "<<i<<" is regular? "<<isRegOct<<std::endl;
}
}//end for i
SetUserContexts(damg);
if(!rank) {
std::cout << "Set User Contexts all levels."<< std::endl;
}
MPI_Barrier(MPI_COMM_WORLD);
PetscInt jacType = 1;
PetscOptionsGetInt(0,"-jacType",&jacType,0);
PetscInt rhsType = 1;
PetscOptionsGetInt(0,"-rhsType",&rhsType,0);
createLmatType2(LaplacianType2Stencil);
createMmatType2(MassType2Stencil);
if(jacType == 3) {
createLmatType1(LaplacianType1Stencil);
createMmatType1(MassType1Stencil);
}
createShFnMat(ShapeFnStencil);
if(!rank) {
std::cout << "Created Stencils."<< std::endl;
}
//Function handles
PetscErrorCode (*ComputeRHSHandle)(ot::DAMG damg,Vec rhs) = NULL;
PetscErrorCode (*CreateJacobianHandle)(ot::DAMG damg,Mat *B) = NULL;
PetscErrorCode (*ComputeJacobianHandle)(ot::DAMG damg,Mat J, Mat B) = NULL;
if(rhsType == 0) {
ComputeRHSHandle = ComputeRHS0;
} else if (rhsType == 1) {
ComputeRHSHandle = ComputeRHS1;
} else if (rhsType == 2) {
ComputeRHSHandle = ComputeRHS2;
} else if (rhsType == 3) {
ComputeRHSHandle = ComputeRHS3;
} else if (rhsType == 4) {
ComputeRHSHandle = ComputeRHS4;
} else if (rhsType == 5) {
ComputeRHSHandle = ComputeRHS5;
} else if (rhsType == 6) {
ComputeRHSHandle = ComputeRHS6;
} else if (rhsType == 7) {
ComputeRHSHandle = ComputeRHS7;
} else if (rhsType == 8) {
ComputeRHSHandle = ComputeRHS8;
} else {
assert(false);
}
if(jacType == 1) {
CreateJacobianHandle = CreateJacobian1;
ComputeJacobianHandle = ComputeJacobian1;
} else if (jacType == 2) {
CreateJacobianHandle = CreateJacobian2;
ComputeJacobianHandle = ComputeJacobian2;
} else if (jacType == 3) {
CreateJacobianHandle = CreateJacobian3;
ComputeJacobianHandle = ComputeJacobian3;
//Skip the finest and the coarsest levels. For the other levels, J and B
//must be different
for(int i = 1; i < (nlevels-1); i++) {
ot::DAMGCreateJMatrix(damg[i], CreateJacobianHandle);
}
} else {
assert(false);
}
//Global Function Handles for using KSP_Shell (will be used @ the coarsest grid if not all
//processors are active on the coarsest grid)
if (jacType == 1) {
ot::getPrivateMatricesForKSP_Shell = getPrivateMatricesForKSP_Shell_Jac1;
} else if (jacType == 2) {
ot::getPrivateMatricesForKSP_Shell = getPrivateMatricesForKSP_Shell_Jac2;
} else if (jacType == 3) {
ot::getPrivateMatricesForKSP_Shell = getPrivateMatricesForKSP_Shell_Jac3;
} else {
assert(false);
}
ot::DAMGSetKSP(damg, CreateJacobianHandle, ComputeJacobianHandle, ComputeRHSHandle);
if(!rank) {
std::cout<<"Solving u-Lu=f"<<std::endl;
}
iC(ot::DAMGSolve(damg));
destroyLmatType2(LaplacianType2Stencil);
destroyMmatType2(MassType2Stencil);
if(jacType == 3) {
destroyLmatType1(LaplacianType1Stencil);
destroyMmatType1(MassType1Stencil);
}
destroyShFnMat(ShapeFnStencil);
MPI_Barrier(MPI_COMM_WORLD);
DestroyUserContexts(damg);
if (!rank) {
std::cout << GRN << "Destroyed User Contexts." << NRM << std::endl;
}
MPI_Barrier(MPI_COMM_WORLD);
iC(DAMGDestroy(damg));
if (!rank) {
std::cout << GRN << "Destroyed DAMG" << NRM << std::endl;
}
#ifdef PETSC_USE_LOG
PetscLogStagePop();
#endif
balOct.clear();
if (!rank) {
std::cout << GRN << "Finalizing ..." << NRM << std::endl;
}
ot::DAMG_Finalize();
PetscFinalize();
}//end function
int main(int argc,char ** argv)
{
PetscErrorCode ierr;
char pfdata_file[PETSC_MAX_PATH_LEN]="datafiles/case9.m";
PFDATA *pfdata;
PetscInt numEdges=0,numVertices=0;
int *edges = NULL;
PetscInt i;
DM networkdm;
PetscInt componentkey[4];
UserCtx User;
PetscLogStage stage1,stage2;
PetscMPIInt size,rank;
PetscInt eStart, eEnd, vStart, vEnd,j;
PetscInt genj,loadj;
Vec X,F;
Mat J;
SNES snes;
ierr = PetscInitialize(&argc,&argv,"pfoptions",help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
{
/* introduce the const crank so the clang static analyzer realizes that if it enters any of the if (crank) then it must have entered the first */
/* this is an experiment to see how the analyzer reacts */
const PetscMPIInt crank = rank;
/* Create an empty network object */
ierr = DMNetworkCreate(PETSC_COMM_WORLD,&networkdm);CHKERRQ(ierr);
/* Register the components in the network */
ierr = DMNetworkRegisterComponent(networkdm,"branchstruct",sizeof(struct _p_EDGEDATA),&componentkey[0]);CHKERRQ(ierr);
ierr = DMNetworkRegisterComponent(networkdm,"busstruct",sizeof(struct _p_VERTEXDATA),&componentkey[1]);CHKERRQ(ierr);
ierr = DMNetworkRegisterComponent(networkdm,"genstruct",sizeof(struct _p_GEN),&componentkey[2]);CHKERRQ(ierr);
ierr = DMNetworkRegisterComponent(networkdm,"loadstruct",sizeof(struct _p_LOAD),&componentkey[3]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Read Data",&stage1);CHKERRQ(ierr);
PetscLogStagePush(stage1);
/* READ THE DATA */
if (!crank) {
/* READ DATA */
/* Only rank 0 reads the data */
ierr = PetscOptionsGetString(NULL,NULL,"-pfdata",pfdata_file,PETSC_MAX_PATH_LEN-1,NULL);CHKERRQ(ierr);
ierr = PetscNew(&pfdata);CHKERRQ(ierr);
ierr = PFReadMatPowerData(pfdata,pfdata_file);CHKERRQ(ierr);
User.Sbase = pfdata->sbase;
numEdges = pfdata->nbranch;
numVertices = pfdata->nbus;
ierr = PetscMalloc(2*numEdges*sizeof(int),&edges);CHKERRQ(ierr);
ierr = GetListofEdges(pfdata->nbranch,pfdata->branch,edges);CHKERRQ(ierr);
}
PetscLogStagePop();
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Create network",&stage2);CHKERRQ(ierr);
PetscLogStagePush(stage2);
/* Set number of nodes/edges */
ierr = DMNetworkSetSizes(networkdm,numVertices,numEdges,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
/* Add edge connectivity */
ierr = DMNetworkSetEdgeList(networkdm,edges);CHKERRQ(ierr);
/* Set up the network layout */
ierr = DMNetworkLayoutSetUp(networkdm);CHKERRQ(ierr);
if (!crank) {
ierr = PetscFree(edges);CHKERRQ(ierr);
}
/* Add network components only process 0 has any data to add*/
if (!crank) {
genj=0; loadj=0;
ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr);
for (i = eStart; i < eEnd; i++) {
ierr = DMNetworkAddComponent(networkdm,i,componentkey[0],&pfdata->branch[i-eStart]);CHKERRQ(ierr);
}
ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr);
for (i = vStart; i < vEnd; i++) {
ierr = DMNetworkAddComponent(networkdm,i,componentkey[1],&pfdata->bus[i-vStart]);CHKERRQ(ierr);
if (pfdata->bus[i-vStart].ngen) {
for (j = 0; j < pfdata->bus[i-vStart].ngen; j++) {
ierr = DMNetworkAddComponent(networkdm,i,componentkey[2],&pfdata->gen[genj++]);CHKERRQ(ierr);
}
}
if (pfdata->bus[i-vStart].nload) {
for (j=0; j < pfdata->bus[i-vStart].nload; j++) {
ierr = DMNetworkAddComponent(networkdm,i,componentkey[3],&pfdata->load[loadj++]);CHKERRQ(ierr);
}
}
/* Add number of variables */
ierr = DMNetworkAddNumVariables(networkdm,i,2);CHKERRQ(ierr);
}
}
/* Set up DM for use */
ierr = DMSetUp(networkdm);CHKERRQ(ierr);
if (!crank) {
ierr = PetscFree(pfdata->bus);CHKERRQ(ierr);
ierr = PetscFree(pfdata->gen);CHKERRQ(ierr);
ierr = PetscFree(pfdata->branch);CHKERRQ(ierr);
ierr = PetscFree(pfdata->load);CHKERRQ(ierr);
ierr = PetscFree(pfdata);CHKERRQ(ierr);
}
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size > 1) {
DM distnetworkdm;
/* Network partitioning and distribution of data */
ierr = DMNetworkDistribute(networkdm,0,&distnetworkdm);CHKERRQ(ierr);
ierr = DMDestroy(&networkdm);CHKERRQ(ierr);
networkdm = distnetworkdm;
}
PetscLogStagePop();
ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr);
ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr);
#if 0
PetscInt numComponents;
EDGEDATA edge;
PetscInt offset,key,kk;
DMNetworkComponentGenericDataType *arr;
VERTEXDATA bus;
GEN gen;
LOAD load;
for (i = eStart; i < eEnd; i++) {
ierr = DMNetworkGetComponentDataArray(networkdm,&arr);CHKERRQ(ierr);
ierr = DMNetworkGetComponentTypeOffset(networkdm,i,0,&key,&offset);CHKERRQ(ierr);
edge = (EDGEDATA)(arr+offset);
ierr = DMNetworkGetNumComponents(networkdm,i,&numComponents);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d ncomps = %d Line %d ---- %d\n",crank,numComponents,edge->internal_i,edge->internal_j);CHKERRQ(ierr);
}
for (i = vStart; i < vEnd; i++) {
ierr = DMNetworkGetComponentDataArray(networkdm,&arr);CHKERRQ(ierr);
ierr = DMNetworkGetNumComponents(networkdm,i,&numComponents);CHKERRQ(ierr);
for (kk=0; kk < numComponents; kk++) {
ierr = DMNetworkGetComponentTypeOffset(networkdm,i,kk,&key,&offset);CHKERRQ(ierr);
if (key == 1) {
bus = (VERTEXDATA)(arr+offset);
ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d ncomps = %d Bus %d\n",crank,numComponents,bus->internal_i);CHKERRQ(ierr);
} else if (key == 2) {
gen = (GEN)(arr+offset);
ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d Gen pg = %f qg = %f\n",crank,gen->pg,gen->qg);CHKERRQ(ierr);
} else if (key == 3) {
load = (LOAD)(arr+offset);
ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d Load pl = %f ql = %f\n",crank,load->pl,load->ql);CHKERRQ(ierr);
}
}
}
#endif
/* Broadcast Sbase to all processors */
ierr = MPI_Bcast(&User.Sbase,1,MPIU_SCALAR,0,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(networkdm,&X);CHKERRQ(ierr);
ierr = VecDuplicate(X,&F);CHKERRQ(ierr);
ierr = DMCreateMatrix(networkdm,&J);CHKERRQ(ierr);
ierr = MatSetOption(J,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = SetInitialValues(networkdm,X,&User);CHKERRQ(ierr);
/* HOOK UP SOLVER */
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes,networkdm);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,F,FormFunction,&User);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,J,J,FormJacobian,&User);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = SNESSolve(snes,NULL,X);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&networkdm);CHKERRQ(ierr);
}
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
PetscBool flg,TestOverlap,TestSubMat,TestAllcols,test_sorted=PETSC_FALSE;
PetscInt vid = -1;
#if defined(PETSC_USE_LOG)
PetscLogStage stages[2];
#endif
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_sorted",&test_sorted,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<Mbs; i++) {
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatEqual(A, Atrans, &flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
if (vid >= 0 && vid < size) {
if (!rank) printf("A: \n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("sA: \n");
ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test sA==A through MatMult() */
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA");
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
if (!TestAllcols) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
if (rank == vid) {
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
} else { /* Test all rows and colums */
sz = M;
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr);
if (rank == vid) {
PetscBool colflag;
ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr);
printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
}
}
ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);CHKERRQ(ierr);
/* Test MatIncreaseOverlap */
if (TestOverlap) {
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (rank == vid) {
printf("\n[%d] IS from BAIJ:\n",rank);
ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
printf("\n[%d] IS from SBAIJ:\n",rank);
ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
if (!rank) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
ierr = ISSort(is2[i]);CHKERRQ(ierr);
/* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
}
SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i);
}
}
}
/* Test MatCreateSubmatrices */
if (TestSubMat) {
if (test_sorted) {
for (i = 0; i < nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
}
}
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA");
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatCreateSubmatrices(): A != sA");
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatsA);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
//STARTBDRYRESIDUALS
PetscErrorCode FormFunction(SNES snes, Vec u, Vec F, void *ctx) {
PetscErrorCode ierr;
unfemCtx *user = (unfemCtx*)ctx;
const int *abfn, *ae, *as, *abfs, *en, deg = user->quaddeg - 1;
const Node *aloc;
const double *au;
double *aF, unode[3], gradu[2], gradpsi[3][2], uquad[4], aquad[4],
fquad[4], dx, dy, dx1, dx2, dy1, dy2, detJ,
ls, xmid, ymid, sint, xx, yy, sum;
int n, p, na, nb, k, l, q;
PetscLogStagePush(user->resstage); //STRIP
ierr = VecGetArrayRead(u,&au); CHKERRQ(ierr);
ierr = VecSet(F,0.0); CHKERRQ(ierr);
ierr = VecGetArray(F,&aF); CHKERRQ(ierr);
ierr = UMGetNodeCoordArrayRead(user->mesh,&aloc); CHKERRQ(ierr);
// Dirichlet node residuals
ierr = ISGetIndices(user->mesh->bfn,&abfn); CHKERRQ(ierr);
for (n = 0; n < user->mesh->N; n++) {
if (abfn[n] == 2) // node is Dirichlet
aF[n] = au[n] - user->gD_fcn(aloc[n].x,aloc[n].y);
}
// Neumann segment contributions
ierr = ISGetIndices(user->mesh->s,&as); CHKERRQ(ierr);
ierr = ISGetIndices(user->mesh->bfs,&abfs); CHKERRQ(ierr);
for (p = 0; p < user->mesh->P; p++) {
if (abfs[p] == 1) { // segment is Neumann
na = as[2*p+0]; // nodes at end of segment
nb = as[2*p+1];
// length of segment
dx = aloc[na].x-aloc[nb].x;
dy = aloc[na].y-aloc[nb].y;
ls = sqrt(dx * dx + dy * dy);
// midpoint rule; psi_na=psi_nb=0.5 at midpoint of segment
xmid = 0.5*(aloc[na].x+aloc[nb].x);
ymid = 0.5*(aloc[na].y+aloc[nb].y);
sint = 0.5 * user->gN_fcn(xmid,ymid) * ls;
// nodes at end of segment could be Dirichlet
if (abfn[na] != 2)
aF[na] -= sint;
if (abfn[nb] != 2)
aF[nb] -= sint;
}
}
ierr = ISRestoreIndices(user->mesh->s,&as); CHKERRQ(ierr);
ierr = ISRestoreIndices(user->mesh->bfs,&abfs); CHKERRQ(ierr);
//ENDBDRYRESIDUALS
//STARTELEMENTRESIDUALS
// element contributions
ierr = ISGetIndices(user->mesh->e,&ae); CHKERRQ(ierr);
for (k = 0; k < user->mesh->K; k++) {
en = ae + 3*k; // en[0], en[1], en[2] are nodes of element k
// geometry of element
dx1 = aloc[en[1]].x - aloc[en[0]].x;
dx2 = aloc[en[2]].x - aloc[en[0]].x;
dy1 = aloc[en[1]].y - aloc[en[0]].y;
dy2 = aloc[en[2]].y - aloc[en[0]].y;
detJ = dx1 * dy2 - dx2 * dy1;
// gradients of hat functions
for (l = 0; l < 3; l++) {
gradpsi[l][0] = ( dy2 * dchi[l][0] - dy1 * dchi[l][1]) / detJ;
gradpsi[l][1] = (-dx2 * dchi[l][0] + dx1 * dchi[l][1]) / detJ;
}
// u and grad u on element
gradu[0] = 0.0;
gradu[1] = 0.0;
for (l = 0; l < 3; l++) {
if (abfn[en[l]] == 2)
unode[l] = user->gD_fcn(aloc[en[l]].x,aloc[en[l]].y);
else
unode[l] = au[en[l]];
gradu[0] += unode[l] * gradpsi[l][0];
gradu[1] += unode[l] * gradpsi[l][1];
}
// function values at quadrature points on element
for (q = 0; q < Q[deg]; q++) {
uquad[q] = eval(unode,xi[deg][q],eta[deg][q]);
xx = aloc[en[0]].x + dx1 * xi[deg][q] + dx2 * eta[deg][q];
yy = aloc[en[0]].y + dy1 * xi[deg][q] + dy2 * eta[deg][q];
aquad[q] = user->a_fcn(uquad[q],xx,yy);
fquad[q] = user->f_fcn(uquad[q],xx,yy);
}
// residual contribution for each node of element
for (l = 0; l < 3; l++) {
if (abfn[en[l]] < 2) { // if NOT a Dirichlet node
sum = 0.0;
for (q = 0; q < Q[deg]; q++)
sum += w[deg][q]
* ( aquad[q] * InnerProd(gradu,gradpsi[l])
- fquad[q] * chi(l,xi[deg][q],eta[deg][q]) );
aF[en[l]] += fabs(detJ) * sum;
}
}
}
ierr = ISRestoreIndices(user->mesh->e,&ae); CHKERRQ(ierr);
ierr = ISRestoreIndices(user->mesh->bfn,&abfn); CHKERRQ(ierr);
ierr = UMRestoreNodeCoordArrayRead(user->mesh,&aloc); CHKERRQ(ierr);
ierr = VecRestoreArrayRead(u,&au); CHKERRQ(ierr);
ierr = VecRestoreArray(F,&aF); CHKERRQ(ierr);
PetscLogStagePop(); //STRIP
return 0;
}
int main(int argc,char **argv)
{
DM da; /* distributed array */
Vec x,b,u; /* approx solution, RHS, exact solution */
Mat A; /* linear system matrix */
KSP ksp; /* linear solver context */
PetscRandom rctx; /* random number generator context */
PetscReal norm; /* norm of solution error */
PetscInt i,j,its;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE;
PetscLogStage stage;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/*
Create distributed array to handle parallel distribution.
The problem size will default to 8 by 7, but this can be
changed using -da_grid_x M -da_grid_y N
*/
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-8,-7,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create parallel matrix preallocated according to the DMDA, format AIJ by default.
To use symmetric storage, run with -dm_mat_type sbaij -mat_ignore_lower_triangular
*/
ierr = DMSetMatType(da,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
/*
Set matrix elements for the 2-D, five-point stencil in parallel.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Rows and columns are specified by the stencil
- Entries are normalized for a domain [0,1]x[0,1]
*/
ierr = PetscLogStageRegister("Assembly", &stage);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
for (j=info.ys; j<info.ys+info.ym; j++) {
for (i=info.xs; i<info.xs+info.xm; i++) {
PetscReal hx = 1./info.mx,hy = 1./info.my;
MatStencil row = {0},col[5] = {{0}};
PetscScalar v[5];
PetscInt ncols = 0;
row.j = j; row.i = i;
col[ncols].j = j; col[ncols].i = i; v[ncols++] = 2*(hx/hy + hy/hx);
/* boundaries */
if (i>0) {col[ncols].j = j; col[ncols].i = i-1; v[ncols++] = -hy/hx;}
if (i<info.mx-1) {col[ncols].j = j; col[ncols].i = i+1; v[ncols++] = -hy/hx;}
if (j>0) {col[ncols].j = j-1; col[ncols].i = i; v[ncols++] = -hx/hy;}
if (j<info.my-1) {col[ncols].j = j+1; col[ncols].i = i; v[ncols++] = -hx/hy;}
ierr = MatSetValuesStencil(A,1,&row,ncols,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd()
Computations can be done while messages are in transition
by placing code between these two statements.
*/
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
/*
Create parallel vectors compatible with the DMDA.
*/
ierr = DMCreateGlobalVector(da,&u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b);CHKERRQ(ierr);
ierr = VecDuplicate(u,&x);CHKERRQ(ierr);
/*
Set exact solution; then compute right-hand-side vector.
By default we use an exact solution of a vector with all
elements of 1.0; Alternatively, using the runtime option
-random_sol forms a solution vector with random components.
*/
ierr = PetscOptionsGetBool(NULL,"-random_exact_sol",&flg,NULL);CHKERRQ(ierr);
if (flg) {
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
ierr = VecSetRandom(u,rctx);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
} else {
ierr = VecSet(u,1.);CHKERRQ(ierr);
}
ierr = MatMult(A,u,b);CHKERRQ(ierr);
/*
View the exact solution vector if desired
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-view_exact_sol",&flg,NULL);CHKERRQ(ierr);
if (flg) {ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
These options will override those specified above as long as
KSPSetFromOptions() is called _after_ any other customization
routines.
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Check solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Check the error
*/
ierr = VecAXPY(x,-1.,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
/*
Print convergence information. PetscPrintf() produces a single
print statement from all processes that share a communicator.
An alternative is PetscFPrintf(), which prints to a file.
*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g iterations %D\n",(double)norm,its);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
ierr = PetscFinalize();
return 0;
}
/*
* time_step solves for the time_dependence of the system
* that was previously setup using the add_to_ham and add_lin
* routines. Solver selection and parameters can be controlled via PETSc
* command line options. Default solver is TSRK3BS
*
* Inputs:
* Vec x: The density matrix, with appropriate inital conditions
* double dt: initial timestep. For certain explicit methods, this timestep
* can be changed, as those methods have adaptive time steps
* double time_max: the maximum time to integrate to
* int steps_max: max number of steps to take
*/
void time_step(Vec x, PetscReal init_time, PetscReal time_max,PetscReal dt,PetscInt steps_max){
PetscViewer mat_view;
TS ts; /* timestepping context */
PetscInt i,j,Istart,Iend,steps,row,col;
PetscScalar mat_tmp;
PetscReal tmp_real;
Mat AA;
PetscInt nevents,direction;
PetscBool terminate;
operator op;
int num_pop;
double *populations;
Mat solve_A,solve_stiff_A;
PetscLogStagePop();
PetscLogStagePush(solve_stage);
if (_lindblad_terms) {
if (nid==0) {
printf("Lindblad terms found, using Lindblad solver.\n");
}
solve_A = full_A;
if (_stiff_solver) {
if(nid==0) printf("ERROR! Lindblad-stiff solver untested.");
exit(0);
}
} else {
if (nid==0) {
printf("No Lindblad terms found, using (more efficient) Schrodinger solver.\n");
}
solve_A = ham_A;
solve_stiff_A = ham_stiff_A;
if (_num_time_dep&&_stiff_solver) {
if(nid==0) printf("ERROR! Schrodinger-stiff + timedep solver untested.");
exit(0);
}
}
/* Possibly print dense ham. No stabilization is needed? */
if (nid==0) {
/* Print dense ham, if it was asked for */
if (_print_dense_ham){
FILE *fp_ham;
fp_ham = fopen("ham","w");
if (nid==0){
for (i=0;i<total_levels;i++){
for (j=0;j<total_levels;j++){
fprintf(fp_ham,"%e %e ",PetscRealPart(_hamiltonian[i][j]),PetscImaginaryPart(_hamiltonian[i][j]));
}
fprintf(fp_ham,"\n");
}
}
fclose(fp_ham);
for (i=0;i<total_levels;i++){
free(_hamiltonian[i]);
}
free(_hamiltonian);
_print_dense_ham = 0;
}
}
/* Remove stabilization if it was previously added */
if (stab_added){
if (nid==0) printf("Removing stabilization...\n");
/*
* We add 1.0 in the 0th spot and every n+1 after
*/
if (nid==0) {
row = 0;
for (i=0;i<total_levels;i++){
col = i*(total_levels+1);
mat_tmp = -1.0 + 0.*PETSC_i;
MatSetValue(full_A,row,col,mat_tmp,ADD_VALUES);
}
}
}
MatGetOwnershipRange(solve_A,&Istart,&Iend);
/*
* Explicitly add 0.0 to all diagonal elements;
* this fixes a 'matrix in wrong state' message that PETSc
* gives if the diagonal was never initialized.
*/
//if (nid==0) printf("Adding 0 to diagonal elements...\n");
for (i=Istart;i<Iend;i++){
mat_tmp = 0 + 0.*PETSC_i;
MatSetValue(solve_A,i,i,mat_tmp,ADD_VALUES);
}
if(_stiff_solver){
MatGetOwnershipRange(solve_stiff_A,&Istart,&Iend);
for (i=Istart;i<Iend;i++){
mat_tmp = 0 + 0.*PETSC_i;
MatSetValue(solve_stiff_A,i,i,mat_tmp,ADD_VALUES);
}
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*
* Create the timestepping solver and set various options *
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
* Create timestepping solver context
*/
TSCreate(PETSC_COMM_WORLD,&ts);
TSSetProblemType(ts,TS_LINEAR);
/*
* Set function to get information at every timestep
*/
if (_ts_monitor!=NULL){
TSMonitorSet(ts,_ts_monitor,_tsctx,NULL);
}
/*
* Set up ODE system
*/
TSSetRHSFunction(ts,NULL,TSComputeRHSFunctionLinear,NULL);
if(_stiff_solver) {
/* TSSetIFunction(ts,NULL,TSComputeRHSFunctionLinear,NULL); */
if (nid==0) {
printf("Stiff solver not implemented!\n");
exit(0);
}
if(nid==0) printf("Using stiff solver - TSROSW\n");
}
if(_num_time_dep+_num_time_dep_lin) {
for(i=0;i<_num_time_dep;i++){
tmp_real = 0.0;
_add_ops_to_mat_ham(tmp_real,solve_A,_time_dep_list[i].num_ops,_time_dep_list[i].ops);
}
for(i=0;i<_num_time_dep_lin;i++){
tmp_real = 0.0;
_add_ops_to_mat_lin(tmp_real,solve_A,_time_dep_list_lin[i].num_ops,_time_dep_list_lin[i].ops);
}
/* Tell PETSc to assemble the matrix */
MatAssemblyBegin(solve_A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(solve_A,MAT_FINAL_ASSEMBLY);
if (nid==0) printf("Matrix Assembled.\n");
MatDuplicate(solve_A,MAT_COPY_VALUES,&AA);
MatAssemblyBegin(AA,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(AA,MAT_FINAL_ASSEMBLY);
TSSetRHSJacobian(ts,AA,AA,_RHS_time_dep_ham_p,NULL);
} else {
/* Tell PETSc to assemble the matrix */
MatAssemblyBegin(solve_A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(solve_A,MAT_FINAL_ASSEMBLY);
if (_stiff_solver){
MatAssemblyBegin(solve_stiff_A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(solve_stiff_A,MAT_FINAL_ASSEMBLY);
/* TSSetIJacobian(ts,solve_stiff_A,solve_stiff_A,TSComputeRHSJacobianConstant,NULL); */
if (nid==0) {
printf("Stiff solver not implemented!\n");
exit(0);
}
}
if (nid==0) printf("Matrix Assembled.\n");
TSSetRHSJacobian(ts,solve_A,solve_A,TSComputeRHSJacobianConstant,NULL);
}
/* Print information about the matrix. */
PetscViewerASCIIOpen(PETSC_COMM_WORLD,NULL,&mat_view);
PetscViewerPushFormat(mat_view,PETSC_VIEWER_ASCII_INFO);
/* PetscViewerPushFormat(mat_view,PETSC_VIEWER_ASCII_MATLAB); */
/* MatView(solve_A,mat_view); */
/* PetscInt ncols; */
/* const PetscInt *cols; */
/* const PetscScalar *vals; */
/* for(i=0;i<total_levels*total_levels;i++){ */
/* MatGetRow(solve_A,i,&ncols,&cols,&vals); */
/* for (j=0;j<ncols;j++){ */
/* if(PetscAbsComplex(vals[j])>1e-5){ */
/* printf("%d %d %lf %lf\n",i,cols[j],vals[j]); */
/* } */
/* } */
/* MatRestoreRow(solve_A,i,&ncols,&cols,&vals); */
/* } */
if(_stiff_solver){
MatView(solve_stiff_A,mat_view);
}
PetscViewerPopFormat(mat_view);
PetscViewerDestroy(&mat_view);
TSSetTimeStep(ts,dt);
/*
* Set default options, can be changed at runtime
*/
TSSetMaxSteps(ts,steps_max);
TSSetMaxTime(ts,time_max);
TSSetTime(ts,init_time);
TSSetExactFinalTime(ts,TS_EXACTFINALTIME_STEPOVER);
if (_stiff_solver) {
TSSetType(ts,TSROSW);
} else {
TSSetType(ts,TSRK);
TSRKSetType(ts,TSRK3BS);
}
/* If we have gates to apply, set up the event handler. */
if (_num_quantum_gates > 0) {
nevents = 1; //Only one event for now (did we cross a gate?)
direction = -1; //We only want to count an event if we go from positive to negative
terminate = PETSC_FALSE; //Keep time stepping after we passed our event
/* Arguments are: ts context, nevents, direction of zero crossing, whether to terminate,
* a function to check event status, a function to apply events, private data context.
*/
TSSetEventHandler(ts,nevents,&direction,&terminate,_QG_EventFunction,_QG_PostEventFunction,NULL);
}
if (_num_circuits > 0) {
nevents = 1; //Only one event for now (did we cross a gate?)
direction = -1; //We only want to count an event if we go from positive to negative
terminate = PETSC_FALSE; //Keep time stepping after we passed our event
/* Arguments are: ts context, nevents, direction of zero crossing, whether to terminate,
* a function to check event status, a function to apply events, private data context.
*/
TSSetEventHandler(ts,nevents,&direction,&terminate,_QC_EventFunction,_QC_PostEventFunction,NULL);
}
if (_discrete_ec > 0) {
nevents = 1; //Only one event for now (did we cross an ec step?)
direction = -1; //We only want to count an event if we go from positive to negative
terminate = PETSC_FALSE; //Keep time stepping after we passed our event
/* Arguments are: ts context, nevents, direction of zero crossing, whether to terminate,
* a function to check event status, a function to apply events, private data context.
*/
TSSetEventHandler(ts,nevents,&direction,&terminate,_DQEC_EventFunction,_DQEC_PostEventFunction,NULL);
}
/* if (_lindblad_terms) { */
/* nevents = 1; //Only one event for now (did we cross a gate?) */
/* direction = 0; //We only want to count an event if we go from positive to negative */
/* terminate = PETSC_FALSE; //Keep time stepping after we passed our event */
/* TSSetEventHandler(ts,nevents,&direction,&terminate,_Normalize_EventFunction,_Normalize_PostEventFunction,NULL); */
/* } */
TSSetFromOptions(ts);
TSSolve(ts,x);
TSGetStepNumber(ts,&steps);
num_pop = get_num_populations();
populations = malloc(num_pop*sizeof(double));
get_populations(x,&populations);
/* if(nid==0){ */
/* printf("Final populations: "); */
/* for(i=0;i<num_pop;i++){ */
/* printf(" %e ",populations[i]); */
/* } */
/* printf("\n"); */
/* } */
/* PetscPrintf(PETSC_COMM_WORLD,"Steps %D\n",steps); */
/* Free work space */
TSDestroy(&ts);
if(_num_time_dep+_num_time_dep_lin){
MatDestroy(&AA);
}
free(populations);
PetscLogStagePop();
PetscLogStagePush(post_solve_stage);
return;
}
int main(int argc, char **argv){
PetscErrorCode ierr;
int nx = 63, ny = 63;
DM dm;
PetscBool flg;
Mat A;
Vec u, b;
KSP solver;
PC pc;
double norm;
PetscInt stage;
ierr = PetscInitialize(&argc, &argv, NULL, NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-nx", &nx, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-ny", &ny, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL, "-assemble", &flg);CHKERRQ(ierr);
ierr = PetscLogStageRegister("preparing",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Domain creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = createDomain(&dm, nx, ny);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("matrix creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = createMat(dm, &A, flg);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Vector creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm, &b);CHKERRQ(ierr);
ierr = VecDuplicate(b, &u);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Domain initialisation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = init2d(dm, b);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("solver creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD, &solver);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(solver, "poisson_");CHKERRQ(ierr);
ierr = KSPSetOperators(solver, A, A, DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetType(solver, KSPCG);
ierr = KSPGetPC(solver, &pc);CHKERRQ(ierr);
ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr);
ierr = KSPSetFromOptions(solver);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Solving",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = KSPSolve(solver, b, u);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
//ierr = VecView(u, PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
//sleep(10);
VecDestroy(&u);
VecDestroy(&b);
MatDestroy(&A);
DMDestroy(&dm);
KSPDestroy(&solver);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode TestVecClosure(DM dm, PetscBool useIndex, PetscBool useSpectral, AppCtx *user)
{
PetscSection s;
Vec v;
PetscInt numRuns, cStart, cEnd, c, i;
PetscScalar tmpArray[64];
PetscScalar *userArray = user->reuseArray ? tmpArray : NULL;
PetscReal maxTimePerRun = user->maxVecClosureTime;
PetscLogStage stage;
PetscLogEvent event;
PetscEventPerfInfo eventInfo;
PetscErrorCode ierr;
PetscFunctionBegin;
if (useIndex) {
if (useSpectral) {
ierr = PetscLogStageRegister("DMPlex Vector Closure with Index Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("VecClosureInd", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
} else {
ierr = PetscLogStageRegister("DMPlex Vector Spectral Closure with Index Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("VecClosureSpecInd", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
}
} else {
if (useSpectral) {
ierr = PetscLogStageRegister("DMPlex Vector Spectral Closure Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("VecClosureSpec", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
} else {
ierr = PetscLogStageRegister("DMPlex Vector Closure Test", &stage);CHKERRQ(ierr);
ierr = PetscLogEventRegister("VecClosure", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
}
}
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = DMPlexCreateSection(dm, user->dim, user->numFields, user->numComponents, user->numDof, 0, NULL, NULL, NULL, NULL, &s);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dm, s);CHKERRQ(ierr);
if (useIndex) {ierr = DMPlexCreateClosureIndex(dm, s);CHKERRQ(ierr);}
if (useSpectral) {ierr = DMPlexCreateSpectralClosurePermutation(dm, s);CHKERRQ(ierr);}
ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &v);CHKERRQ(ierr);
ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr);
for (i = 0; i < user->iterations; ++i) {
for (c = cStart; c < cEnd; ++c) {
PetscScalar *closure = userArray;
PetscInt closureSize = 64;
ierr = DMPlexVecGetClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr);
if (!user->reuseArray) {ierr = DMPlexVecRestoreClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr);}
}
}
ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &v);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
numRuns = (cEnd-cStart) * user->iterations;
if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1);
if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0);
if (eventInfo.time > maxTimePerRun * numRuns) {
const char *title = "VecClosures";
const char *titleIndex = "VecClosures with Index";
const char *titleSpec = "VecClosures Spectral";
const char *titleSpecIndex = "VecClosures Spectral with Index";
ierr = PetscPrintf(PETSC_COMM_SELF, "%s: %d Average time per vector closure: %gs standard: %gs\n", useIndex ? (useSpectral ? titleSpecIndex : titleIndex) : (useSpectral ? titleSpec : title), numRuns, eventInfo.time/numRuns, maxTimePerRun);
if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for vector closure %g > standard %g", eventInfo.time/numRuns, maxTimePerRun);
}
PetscFunctionReturn(0);
}
PetscErrorCode FormPicard(SNES snes, Vec u, Mat A, Mat P, void *ctx) {
PetscErrorCode ierr;
unfemCtx *user = (unfemCtx*)ctx;
const int *abfn, *ae, *en, deg = user->quaddeg - 1;
const Node *aloc;
const double *au;
double unode[3], gradpsi[3][2],
uquad[4], aquad[4], v[9],
dx1, dx2, dy1, dy2, detJ, xx, yy, sum;
int n, k, l, m, q, cr, cv, row[3];
PetscLogStagePush(user->jacstage); //STRIP
ierr = MatZeroEntries(P); CHKERRQ(ierr);
ierr = ISGetIndices(user->mesh->bfn,&abfn); CHKERRQ(ierr);
for (n = 0; n < user->mesh->N; n++) {
if (abfn[n] == 2) {
v[0] = 1.0;
ierr = MatSetValues(P,1,&n,1,&n,v,ADD_VALUES); CHKERRQ(ierr);
}
}
ierr = ISGetIndices(user->mesh->e,&ae); CHKERRQ(ierr);
ierr = VecGetArrayRead(u,&au); CHKERRQ(ierr);
ierr = UMGetNodeCoordArrayRead(user->mesh,&aloc); CHKERRQ(ierr);
for (k = 0; k < user->mesh->K; k++) {
en = ae + 3*k; // en[0], en[1], en[2] are nodes of element k
// geometry of element
dx1 = aloc[en[1]].x - aloc[en[0]].x;
dx2 = aloc[en[2]].x - aloc[en[0]].x;
dy1 = aloc[en[1]].y - aloc[en[0]].y;
dy2 = aloc[en[2]].y - aloc[en[0]].y;
detJ = dx1 * dy2 - dx2 * dy1;
// gradients of hat functions and u on element
for (l = 0; l < 3; l++) {
gradpsi[l][0] = ( dy2 * dchi[l][0] - dy1 * dchi[l][1]) / detJ;
gradpsi[l][1] = (-dx2 * dchi[l][0] + dx1 * dchi[l][1]) / detJ;
if (abfn[en[l]] == 2)
unode[l] = user->gD_fcn(aloc[en[l]].x,aloc[en[l]].y);
else
unode[l] = au[en[l]];
}
// function values at quadrature points on element
for (q = 0; q < Q[deg]; q++) {
uquad[q] = eval(unode,xi[deg][q],eta[deg][q]);
xx = aloc[en[0]].x + dx1 * xi[deg][q] + dx2 * eta[deg][q];
yy = aloc[en[0]].y + dy1 * xi[deg][q] + dy2 * eta[deg][q];
aquad[q] = user->a_fcn(uquad[q],xx,yy);
}
// generate 3x3 element stiffness matrix
cr = 0; // count rows
cv = 0; // count values
for (l = 0; l < 3; l++) {
if (abfn[en[l]] != 2) {
row[cr] = en[l];
cr++;
for (m = 0; m < 3; m++) {
if (abfn[en[m]] != 2) {
sum = 0.0;
for (q = 0; q < Q[deg]; q++) {
sum += w[deg][q] * aquad[q]
* InnerProd(gradpsi[l],gradpsi[m]);
}
v[cv] = fabs(detJ) * sum;
cv++;
}
}
}
}
// insert element stiffness matrix
ierr = MatSetValues(P,cr,row,cr,row,v,ADD_VALUES); CHKERRQ(ierr);
}
ierr = ISRestoreIndices(user->mesh->e,&ae); CHKERRQ(ierr);
ierr = ISRestoreIndices(user->mesh->bfn,&abfn); CHKERRQ(ierr);
ierr = VecRestoreArrayRead(u,&au); CHKERRQ(ierr);
ierr = UMRestoreNodeCoordArrayRead(user->mesh,&aloc); CHKERRQ(ierr);
ierr = MatAssemblyBegin(P,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(P,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
if (A != P) {
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
}
ierr = MatSetOption(P,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE); CHKERRQ(ierr);
PetscLogStagePop(); //STRIP
return 0;
}
/*@
PetscConvEstGetConvRate - Returns an estimate of the convergence rate for the discretization
Not collective
Input Parameter:
. ce - The PetscConvEst object
Output Parameter:
. alpha - The convergence rate for each field
Note: The convergence rate alpha is defined by
$ || u_h - u_exact || < C h^alpha
where u_h is the discrete solution, and h is a measure of the discretization size.
We solve a series of problems on refined meshes, calculate an error based upon the exact solution in the DS,
and then fit the result to our model above using linear regression.
Options database keys:
. -snes_convergence_estimate : Execute convergence estimation and print out the rate
Level: intermediate
.keywords: PetscConvEst, convergence
.seealso: PetscConvEstSetSolver(), PetscConvEstCreate(), PetscConvEstGetConvRate()
@*/
PetscErrorCode PetscConvEstGetConvRate(PetscConvEst ce, PetscReal alpha[])
{
DM *dm;
PetscObject disc;
MPI_Comm comm;
const char *uname, *dmname;
void *ctx;
Vec u;
PetscReal t = 0.0, *x, *y, slope, intercept;
PetscInt *dof, dim, Nr = ce->Nr, r, f, oldlevel, oldnlev;
PetscLogEvent event;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) ce, &comm);CHKERRQ(ierr);
ierr = DMGetDimension(ce->idm, &dim);CHKERRQ(ierr);
ierr = DMGetApplicationContext(ce->idm, &ctx);CHKERRQ(ierr);
ierr = DMPlexSetRefinementUniform(ce->idm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMGetRefineLevel(ce->idm, &oldlevel);CHKERRQ(ierr);
ierr = PetscMalloc2((Nr+1), &dm, (Nr+1)*ce->Nf, &dof);CHKERRQ(ierr);
dm[0] = ce->idm;
for (f = 0; f < ce->Nf; ++f) alpha[f] = 0.0;
/* Loop over meshes */
ierr = PetscLogEventRegister("ConvEst Error", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr);
for (r = 0; r <= Nr; ++r) {
PetscLogStage stage;
char stageName[PETSC_MAX_PATH_LEN];
ierr = PetscSNPrintf(stageName, PETSC_MAX_PATH_LEN-1, "ConvEst Refinement Level %D", r);CHKERRQ(ierr);
ierr = PetscLogStageRegister(stageName, &stage);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
if (r > 0) {
ierr = DMRefine(dm[r-1], MPI_COMM_NULL, &dm[r]);CHKERRQ(ierr);
ierr = DMSetCoarseDM(dm[r], dm[r-1]);CHKERRQ(ierr);
ierr = DMCopyDisc(ce->idm, dm[r]);CHKERRQ(ierr);
ierr = DMCopyTransform(ce->idm, dm[r]);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) dm[r-1], &dmname);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm[r], dmname);CHKERRQ(ierr);
for (f = 0; f <= ce->Nf; ++f) {
PetscErrorCode (*nspconstr)(DM, PetscInt, MatNullSpace *);
ierr = DMGetNullSpaceConstructor(dm[r-1], f, &nspconstr);CHKERRQ(ierr);
ierr = DMSetNullSpaceConstructor(dm[r], f, nspconstr);CHKERRQ(ierr);
}
}
ierr = DMViewFromOptions(dm[r], NULL, "-conv_dm_view");CHKERRQ(ierr);
/* Create solution */
ierr = DMCreateGlobalVector(dm[r], &u);CHKERRQ(ierr);
ierr = DMGetField(dm[r], 0, NULL, &disc);CHKERRQ(ierr);
ierr = PetscObjectGetName(disc, &uname);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) u, uname);CHKERRQ(ierr);
/* Setup solver */
ierr = SNESReset(ce->snes);CHKERRQ(ierr);
ierr = SNESSetDM(ce->snes, dm[r]);CHKERRQ(ierr);
ierr = DMPlexSetSNESLocalFEM(dm[r], ctx, ctx, ctx);CHKERRQ(ierr);
ierr = SNESSetFromOptions(ce->snes);CHKERRQ(ierr);
/* Create initial guess */
ierr = DMProjectFunction(dm[r], t, ce->initGuess, ce->ctxs, INSERT_VALUES, u);CHKERRQ(ierr);
ierr = SNESSolve(ce->snes, NULL, u);CHKERRQ(ierr);
ierr = PetscLogEventBegin(event, ce, 0, 0, 0);CHKERRQ(ierr);
ierr = DMComputeL2FieldDiff(dm[r], t, ce->exactSol, ce->ctxs, u, &ce->errors[r*ce->Nf]);CHKERRQ(ierr);
ierr = PetscLogEventEnd(event, ce, 0, 0, 0);CHKERRQ(ierr);
for (f = 0; f < ce->Nf; ++f) {
PetscSection s, fs;
PetscInt lsize;
/* Could use DMGetOutputDM() to add in Dirichlet dofs */
ierr = DMGetSection(dm[r], &s);CHKERRQ(ierr);
ierr = PetscSectionGetField(s, f, &fs);CHKERRQ(ierr);
ierr = PetscSectionGetConstrainedStorageSize(fs, &lsize);CHKERRQ(ierr);
ierr = MPI_Allreduce(&lsize, &dof[r*ce->Nf+f], 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) ce->snes));CHKERRQ(ierr);
ierr = PetscLogEventSetDof(event, f, dof[r*ce->Nf+f]);CHKERRQ(ierr);
ierr = PetscLogEventSetError(event, f, ce->errors[r*ce->Nf+f]);CHKERRQ(ierr);
}
/* Monitor */
if (ce->monitor) {
PetscReal *errors = &ce->errors[r*ce->Nf];
ierr = PetscPrintf(comm, "L_2 Error: ");CHKERRQ(ierr);
if (ce->Nf > 1) {ierr = PetscPrintf(comm, "[");CHKERRQ(ierr);}
for (f = 0; f < ce->Nf; ++f) {
if (f > 0) {ierr = PetscPrintf(comm, ", ");CHKERRQ(ierr);}
if (errors[f] < 1.0e-11) {ierr = PetscPrintf(comm, "< 1e-11");CHKERRQ(ierr);}
else {ierr = PetscPrintf(comm, "%g", (double)errors[f]);CHKERRQ(ierr);}
}
if (ce->Nf > 1) {ierr = PetscPrintf(comm, "]");CHKERRQ(ierr);}
ierr = PetscPrintf(comm, "\n");CHKERRQ(ierr);
}
if (!r) {
/* PCReset() does not wipe out the level structure */
KSP ksp;
PC pc;
ierr = SNESGetKSP(ce->snes, &ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = PCMGGetLevels(pc, &oldnlev);CHKERRQ(ierr);
}
/* Cleanup */
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
}
for (r = 1; r <= Nr; ++r) {
ierr = DMDestroy(&dm[r]);CHKERRQ(ierr);
}
/* Fit convergence rate */
ierr = PetscMalloc2(Nr+1, &x, Nr+1, &y);CHKERRQ(ierr);
for (f = 0; f < ce->Nf; ++f) {
for (r = 0; r <= Nr; ++r) {
x[r] = PetscLog10Real(dof[r*ce->Nf+f]);
y[r] = PetscLog10Real(ce->errors[r*ce->Nf+f]);
}
ierr = PetscLinearRegression(Nr+1, x, y, &slope, &intercept);CHKERRQ(ierr);
/* Since h^{-dim} = N, lg err = s lg N + b = -s dim lg h + b */
alpha[f] = -slope * dim;
}
ierr = PetscFree2(x, y);CHKERRQ(ierr);
ierr = PetscFree2(dm, dof);CHKERRQ(ierr);
/* Restore solver */
ierr = SNESReset(ce->snes);CHKERRQ(ierr);
{
/* PCReset() does not wipe out the level structure */
KSP ksp;
PC pc;
ierr = SNESGetKSP(ce->snes, &ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = PCMGSetLevels(pc, oldnlev, NULL);CHKERRQ(ierr);
ierr = DMSetRefineLevel(ce->idm, oldlevel);CHKERRQ(ierr); /* The damn DMCoarsen() calls in PCMG can reset this */
}
ierr = SNESSetDM(ce->snes, ce->idm);CHKERRQ(ierr);
ierr = DMPlexSetSNESLocalFEM(ce->idm, ctx, ctx, ctx);CHKERRQ(ierr);
ierr = SNESSetFromOptions(ce->snes);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char ** argv ) {
int size, rank;
double startTime, endTime, minTime;
unsigned int incInt = 1;
bool incCorner = 1;
unsigned int numPts;
unsigned int writePOut = 0;
unsigned int writeBOut = 0;
unsigned int readPtsFile = 0;
unsigned int readOctFile =0;
unsigned int runOpt =2;//2 for both, 1 for pt alone and 0 for bal alone.
char Kstr[20];
char inpFileName[50], p2nIn[50], p2nOut[50], n2oOut[50], balOut[50];
double gSize[3];
PetscInitialize(&argc,&argv,"options",NULL);
ot::RegisterEvents();
#ifdef PETSC_USE_LOG
int stages[4];
PetscLogStageRegister("Prepare Input.",&stages[0]);
PetscLogStageRegister("P2O",&stages[1]);
PetscLogStageRegister("N2O",&stages[2]);
PetscLogStageRegister("Bal",&stages[3]);
#endif
MPI_Comm_size(MPI_COMM_WORLD,&size);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
if(argc < 3) {
std::cerr << "Usage: " << argv[0] << " inpfile complx incInt[1] runOpt[2]"<<
"readPtsFile[0] readOctFile[0] maxDepth[30] writePOut[0] writeBOut[0]"<<
" dim[3] maxNumPtsPerOctant[1] incCorner[0] " << std::endl;
return -1;
}
unsigned int maxNumPts= 1;
int complx = atoi(argv[2]);
unsigned int dim=3;
unsigned int maxDepth=30;
if(argc > 3) {
incInt = atoi(argv[3]);
}
if(argc > 4) {
runOpt = atoi(argv[4]);
}
if(argc > 5) {
readPtsFile = atoi(argv[5]);
}
if(argc > 6) {
readOctFile = atoi(argv[6]);
}
if(argc > 7) {
maxDepth = atoi(argv[7]);
}
if(argc > 8) {
writePOut = atoi(argv[8]);
}
if(argc > 9) {
writeBOut = atoi(argv[9]);
}
if(argc > 10) {
dim = atoi(argv[10]);
}
if(argc > 11) {
maxNumPts = atoi(argv[11]);
}
if(argc > 12) { incCorner = (bool)(atoi(argv[12]));}
strcpy(inpFileName, argv[1]);
strcpy(balOut,inpFileName);
ot::int2str(rank,Kstr);
strcat(balOut,Kstr);
strcat(balOut,"_\0");
ot::int2str(size,Kstr);
strcat(balOut,Kstr);
strcpy(n2oOut,balOut);
strcpy(p2nIn,balOut);
strcpy(p2nOut,balOut);
strcat(balOut,"_Bal.ot\0");
strcat(n2oOut,"_Con.ot\0");
strcat(p2nIn,".pts\0");
strcat(p2nOut,"_Out.pts\0");
strcat(inpFileName,".inp\0");
std::vector<ot::TreeNode> nodes;
std::vector<double> pts;
unsigned int ptsLen;
// print out the format ...
/*
if (!rank) {
cout << "============================================================================================" << endl;
cout << "#Points p2n_time p2n_imbal #cellsConstr #cellsBal bal_time bal_imbal maxLevBef minLevBef avgLevBef maxLevAft minLevAft avgLevAft" << endl;
cout << "============================================================================================" << endl << endl;
}
*/
#ifdef PETSC_USE_LOG
PetscLogStagePush(stages[0]);
#endif
if(runOpt == 1 || runOpt == 2) {
if(readPtsFile) {
if(!rank){
std::cout << " reading "<<p2nIn<<std::endl; // Point size
}
ot::readPtsFromFile(p2nIn, pts);
if(!rank){
std::cout << " finished reading "<<p2nIn<<std::endl; // Point size
}
ptsLen = pts.size();
std::vector<ot::TreeNode> tmpNodes;
for(int i = 0; i < ptsLen; i += 3) {
if( (pts[i] > 0.0) &&
(pts[i+1] > 0.0)
&& (pts[i+2] > 0.0) &&
( ((unsigned int)(pts[i]*((double)(1u<<maxDepth)))) < (1u<<maxDepth)) &&
( ((unsigned int)(pts[i+1]*((double)(1u<<maxDepth)))) < (1u<<maxDepth)) &&
( ((unsigned int)(pts[i+2]*((double)(1u<<maxDepth)))) < (1u<<maxDepth)) ) {
tmpNodes.push_back( ot::TreeNode((unsigned int)(pts[i]*(double)(1u<<maxDepth)),
(unsigned int)(pts[i+1]*(double)(1u<<maxDepth)),
(unsigned int)(pts[i+2]*(double)(1u<<maxDepth)),
maxDepth,dim,maxDepth) );
}
}//end for i
pts.clear();
// std::cout<<"Before Removing Duplicates, the number of pts: "<<tmpNodes.size()<<std::endl;
par::removeDuplicates<ot::TreeNode>(tmpNodes,false,MPI_COMM_WORLD);
// std::cout<<"After Removing Duplicates, the number of pts: "<<tmpNodes.size()<<std::endl;
nodes = tmpNodes;
tmpNodes.clear();
par::partitionW<ot::TreeNode>(nodes, NULL,MPI_COMM_WORLD);
// reduce and only print the total ...
DendroIntL locSz, totalSz;
locSz = nodes.size();
par::Mpi_Reduce<DendroIntL>(&locSz, &totalSz, 1, MPI_SUM, 0, MPI_COMM_WORLD);
if(rank==0) {
std::cout<<"rank= "<<rank<<" size= " << size << " total pts= " << totalSz<<std::endl;
}
pts.resize(3*(nodes.size()));
ptsLen = (3*(nodes.size()));
for(int i=0;i<nodes.size();i++) {
pts[3*i] = (((double)(nodes[i].getX())) + 0.5)/((double)(1u<<maxDepth));
pts[(3*i)+1] = (((double)(nodes[i].getY())) +0.5)/((double)(1u<<maxDepth));
pts[(3*i)+2] = (((double)(nodes[i].getZ())) +0.5)/((double)(1u<<maxDepth));
}//end for i
nodes.clear();
}//end if readPts
gSize[0] = 1.0;
gSize[1] = 1.0;
gSize[2] = 1.0;
if(writePOut) {
if(!rank) {
std::cout<<"Writing pts to: "<<p2nOut<<std::endl;
}
ot::writePtsToFile(p2nOut,pts);
}
#ifdef PETSC_USE_LOG
PetscLogStagePop();
#endif
MPI_Barrier(MPI_COMM_WORLD);
#ifdef PETSC_USE_LOG
PetscLogStagePush(stages[1]);
#endif
startTime = MPI_Wtime();
ot::points2Octree(pts, gSize, nodes, dim, maxDepth, maxNumPts, MPI_COMM_WORLD);
endTime = MPI_Wtime();
#ifdef PETSC_USE_LOG
PetscLogStagePop();
#endif
double locTime, totalTime, minTime;
locTime = endTime - startTime;
par::Mpi_Reduce<double>(&locTime, &totalTime, 1, MPI_MAX, 0, MPI_COMM_WORLD);
par::Mpi_Reduce<double>(&locTime, &minTime, 1, MPI_MIN, 0, MPI_COMM_WORLD);
if(!rank){
std::cout <<"P2n Time: "<<totalTime << " " << "secs imbalance: "<< totalTime/minTime << std::endl;
}
pts.clear();
}
//TestBuildOCt from here on:
if(runOpt == 0 || runOpt == 2) {
if(readOctFile) {
ot::readNodesFromFile(n2oOut,nodes);
}
std::vector<ot::TreeNode > linOct;
#ifdef PETSC_USE_LOG
PetscLogStagePush(stages[2]);
#endif
DendroIntL inputNodes = nodes.size();
DendroIntL totInp;
par::Mpi_Reduce<DendroIntL>(&inputNodes, &totInp, 1, MPI_SUM, 0, MPI_COMM_WORLD);
if(!rank) {
std::cout << "Nodes after p2o: "<< totInp << std::endl;
}
ot::completeOctree(nodes,linOct,dim,maxDepth,false, false, false, MPI_COMM_WORLD);
#ifdef PETSC_USE_LOG
PetscLogStagePop();
#endif
assert(!linOct.empty());
if(writeBOut) {
if(!rank) {
std::cout<<"Writing octree after p2o to: "<<n2oOut<<std::endl;
}
ot::writeNodesToFile(n2oOut,linOct);
}
inputNodes = linOct.size();
nodes.clear();
std::vector<ot::TreeNode > balOct;
par::Mpi_Reduce<DendroIntL>(&inputNodes, &totInp, 1, MPI_SUM, 0, MPI_COMM_WORLD);
if(!rank) {
std::cout << "Nodes before balancing: "<< totInp << std::endl;
}
MPI_Barrier(MPI_COMM_WORLD);
#ifdef PETSC_USE_LOG
PetscLogStagePush(stages[3]);
#endif
startTime = MPI_Wtime();
ot::balanceOctree (linOct, balOct, dim, maxDepth, incCorner, MPI_COMM_WORLD, NULL, NULL);
endTime = MPI_Wtime();
#ifdef PETSC_USE_LOG
PetscLogStagePop();
#endif
assert(!balOct.empty());
if(writeBOut) {
if(!rank) {
std::cout<<"Writing octree after balancing to: "<<balOut<<std::endl;
}
ot::writeNodesToFile(balOut,balOct);
}
// compute total inp size and output size
DendroIntL locBalNodes = balOct.size();
DendroIntL totBal;
double balTime, totTime;
balTime = endTime - startTime;
par::Mpi_Reduce<DendroIntL>(&locBalNodes, &totBal, 1, MPI_SUM, 0, MPI_COMM_WORLD);
par::Mpi_Reduce<double>(&balTime, &totTime, 1, MPI_MAX, 0, MPI_COMM_WORLD);
par::Mpi_Reduce<double>(&balTime, &minTime, 1, MPI_MIN, 0, MPI_COMM_WORLD);
if(!rank) {
std::cout << "Nodes after balancing: "<< totBal << std::endl;
std::cout << "bal Time: "<<totTime << " " << "secs imbalance: "<< totTime/minTime << std::endl;
}
linOct.clear();
balOct.clear();
}
PetscFinalize();
}
int main(int argc,char **args)
{
Vec x1,b1,x2,b2; /* solution and RHS vectors for systems #1 and #2 */
Vec u; /* exact solution vector */
Mat C1,C2; /* matrices for systems #1 and #2 */
KSP ksp1,ksp2; /* KSP contexts for systems #1 and #2 */
PetscInt ntimes = 3; /* number of times to solve the linear systems */
PetscLogEvent CHECK_ERROR; /* event number for error checking */
PetscInt ldim,low,high,iglobal,Istart,Iend,Istart2,Iend2;
PetscInt Ii,J,i,j,m = 3,n = 2,its,t;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE;
PetscScalar v;
PetscMPIInt rank,size;
#if defined (PETSC_USE_LOG)
PetscLogStage stages[3];
#endif
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-t",&ntimes,PETSC_NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
n = 2*size;
/*
Register various stages for profiling
*/
ierr = PetscLogStageRegister("Prelim setup",&stages[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Linear System 1",&stages[1]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Linear System 2",&stages[2]);CHKERRQ(ierr);
/*
Register a user-defined event for profiling (error checking).
*/
CHECK_ERROR = 0;
ierr = PetscLogEventRegister("Check Error",KSP_CLASSID,&CHECK_ERROR);CHKERRQ(ierr);
/* - - - - - - - - - - - - Stage 0: - - - - - - - - - - - - - -
Preliminary Setup
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
/*
Create data structures for first linear system.
- Create parallel matrix, specifying only its global dimensions.
When using MatCreate(), the matrix format can be specified at
runtime. Also, the parallel partitioning of the matrix is
determined by PETSc at runtime.
- Create parallel vectors.
- When using VecSetSizes(), we specify only the vector's global
dimension; the parallel partitioning is determined at runtime.
- Note: We form 1 vector from scratch and then duplicate as needed.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&C1);CHKERRQ(ierr);
ierr = MatSetSizes(C1,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr);
ierr = MatSetFromOptions(C1);CHKERRQ(ierr);
ierr = MatSetUp(C1);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(C1,&Istart,&Iend);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = VecSetSizes(u,PETSC_DECIDE,m*n);CHKERRQ(ierr);
ierr = VecSetFromOptions(u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b1);CHKERRQ(ierr);
ierr = VecDuplicate(u,&x1);CHKERRQ(ierr);
/*
Create first linear solver context.
Set runtime options (e.g., -pc_type <type>).
Note that the first linear system uses the default option
names, while the second linear systme uses a different
options prefix.
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp1);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp1);CHKERRQ(ierr);
/*
Set user-defined monitoring routine for first linear system.
*/
ierr = PetscOptionsGetBool(PETSC_NULL,"-my_ksp_monitor",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {ierr = KSPMonitorSet(ksp1,MyKSPMonitor,PETSC_NULL,0);CHKERRQ(ierr);}
/*
Create data structures for second linear system.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&C2);CHKERRQ(ierr);
ierr = MatSetSizes(C2,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr);
ierr = MatSetFromOptions(C2);CHKERRQ(ierr);
ierr = MatSetUp(C2);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(C2,&Istart2,&Iend2);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b2);CHKERRQ(ierr);
ierr = VecDuplicate(u,&x2);CHKERRQ(ierr);
/*
Create second linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp2);CHKERRQ(ierr);
/*
Set different options prefix for second linear system.
Set runtime options (e.g., -s2_pc_type <type>)
*/
ierr = KSPAppendOptionsPrefix(ksp2,"s2_");CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp2);CHKERRQ(ierr);
/*
Assemble exact solution vector in parallel. Note that each
processor needs to set only its local part of the vector.
*/
ierr = VecGetLocalSize(u,&ldim);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(u,&low,&high);CHKERRQ(ierr);
for (i=0; i<ldim; i++) {
iglobal = i + low;
v = (PetscScalar)(i + 100*rank);
ierr = VecSetValues(u,1,&iglobal,&v,ADD_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(u);CHKERRQ(ierr);
ierr = VecAssemblyEnd(u);CHKERRQ(ierr);
/*
Log the number of flops for computing vector entries
*/
ierr = PetscLogFlops(2.0*ldim);CHKERRQ(ierr);
/*
End curent profiling stage
*/
ierr = PetscLogStagePop();CHKERRQ(ierr);
/* --------------------------------------------------------------
Linear solver loop:
Solve 2 different linear systems several times in succession
-------------------------------------------------------------- */
for (t=0; t<ntimes; t++) {
/* - - - - - - - - - - - - Stage 1: - - - - - - - - - - - - - -
Assemble and solve first linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Begin profiling stage #1
*/
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
/*
Initialize all matrix entries to zero. MatZeroEntries() retains
the nonzero structure of the matrix for sparse formats.
*/
if (t > 0) {ierr = MatZeroEntries(C1);CHKERRQ(ierr);}
/*
Set matrix entries in parallel. Also, log the number of flops
for computing matrix entries.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
*/
for (Ii=Istart; Ii<Iend; Ii++) {
v = -1.0; i = Ii/n; j = Ii - i*n;
if (i>0) {J = Ii - n; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C1,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
for (Ii=Istart; Ii<Iend; Ii++) { /* Make matrix nonsymmetric */
v = -1.0*(t+0.5); i = Ii/n;
if (i>0) {J = Ii - n; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
}
ierr = PetscLogFlops(2.0*(Iend-Istart));CHKERRQ(ierr);
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd()
Computations can be done while messages are in transition
by placing code between these two statements.
*/
ierr = MatAssemblyBegin(C1,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C1,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Indicate same nonzero structure of successive linear system matrices
*/
ierr = MatSetOption(C1,MAT_NEW_NONZERO_LOCATIONS,PETSC_TRUE);CHKERRQ(ierr);
/*
Compute right-hand-side vector
*/
ierr = MatMult(C1,u,b1);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
- The flag SAME_NONZERO_PATTERN indicates that the
preconditioning matrix has identical nonzero structure
as during the last linear solve (although the values of
the entries have changed). Thus, we can save some
work in setting up the preconditioner (e.g., no need to
redo symbolic factorization for ILU/ICC preconditioners).
- If the nonzero structure of the matrix is different during
the second linear solve, then the flag DIFFERENT_NONZERO_PATTERN
must be used instead. If you are unsure whether the
matrix structure has changed or not, use the flag
DIFFERENT_NONZERO_PATTERN.
- Caution: If you specify SAME_NONZERO_PATTERN, PETSc
believes your assertion and does not check the structure
of the matrix. If you erroneously claim that the structure
is the same when it actually is not, the new preconditioner
will not function correctly. Thus, use this optimization
feature with caution!
*/
ierr = KSPSetOperators(ksp1,C1,C1,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/*
Use the previous solution of linear system #1 as the initial
guess for the next solve of linear system #1. The user MUST
call KSPSetInitialGuessNonzero() in indicate use of an initial
guess vector; otherwise, an initial guess of zero is used.
*/
if (t>0) {
ierr = KSPSetInitialGuessNonzero(ksp1,PETSC_TRUE);CHKERRQ(ierr);
}
/*
Solve the first linear system. Here we explicitly call
KSPSetUp() for more detailed performance monitoring of
certain preconditioners, such as ICC and ILU. This call
is optional, ase KSPSetUp() will automatically be called
within KSPSolve() if it hasn't been called already.
*/
ierr = KSPSetUp(ksp1);CHKERRQ(ierr);
ierr = KSPSolve(ksp1,b1,x1);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp1,&its);CHKERRQ(ierr);
/*
Check error of solution to first linear system
*/
ierr = CheckError(u,x1,b1,its,1.e-4,CHECK_ERROR);CHKERRQ(ierr);
/* - - - - - - - - - - - - Stage 2: - - - - - - - - - - - - - -
Assemble and solve second linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Conclude profiling stage #1; begin profiling stage #2
*/
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[2]);CHKERRQ(ierr);
/*
Initialize all matrix entries to zero
*/
if (t > 0) {ierr = MatZeroEntries(C2);CHKERRQ(ierr);}
/*
Assemble matrix in parallel. Also, log the number of flops
for computing matrix entries.
- To illustrate the features of parallel matrix assembly, we
intentionally set the values differently from the way in
which the matrix is distributed across the processors. Each
entry that is not owned locally will be sent to the appropriate
processor during MatAssemblyBegin() and MatAssemblyEnd().
- For best efficiency the user should strive to set as many
entries locally as possible.
*/
for (i=0; i<m; i++) {
for (j=2*rank; j<2*rank+2; j++) {
v = -1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
v = 6.0 + t*0.5; ierr = MatSetValues(C2,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
}
for (Ii=Istart2; Ii<Iend2; Ii++) { /* Make matrix nonsymmetric */
v = -1.0*(t+0.5); i = Ii/n;
if (i>0) {J = Ii - n; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
}
ierr = MatAssemblyBegin(C2,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C2,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscLogFlops(2.0*(Iend-Istart));CHKERRQ(ierr);
/*
Indicate same nonzero structure of successive linear system matrices
*/
ierr = MatSetOption(C2,MAT_NEW_NONZERO_LOCATIONS,PETSC_FALSE);CHKERRQ(ierr);
/*
Compute right-hand-side vector
*/
ierr = MatMult(C2,u,b2);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix. Indicate same nonzero
structure of successive preconditioner matrices by setting flag
SAME_NONZERO_PATTERN.
*/
ierr = KSPSetOperators(ksp2,C2,C2,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/*
Solve the second linear system
*/
ierr = KSPSetUp(ksp2);CHKERRQ(ierr);
ierr = KSPSolve(ksp2,b2,x2);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp2,&its);CHKERRQ(ierr);
/*
Check error of solution to second linear system
*/
ierr = CheckError(u,x2,b2,its,1.e-4,CHECK_ERROR);CHKERRQ(ierr);
/*
Conclude profiling stage #2
*/
ierr = PetscLogStagePop();CHKERRQ(ierr);
}
/* --------------------------------------------------------------
End of linear solver loop
-------------------------------------------------------------- */
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = KSPDestroy(&ksp1);CHKERRQ(ierr); ierr = KSPDestroy(&ksp2);CHKERRQ(ierr);
ierr = VecDestroy(&x1);CHKERRQ(ierr); ierr = VecDestroy(&x2);CHKERRQ(ierr);
ierr = VecDestroy(&b1);CHKERRQ(ierr); ierr = VecDestroy(&b2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr); ierr = MatDestroy(&C2);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc, char *argv[]) {
PetscReal (*fList[])(Point) = {funConst, funSin, funStep};
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscReal m=0.0,n=4.0,k=0.0,l=4.0,h=0.1;
PetscInt xSubs = 4, ySubs = 4;
PetscReal mi=1e-3,ro=4,beta=1.5,M=1;
PetscInitialize(&argc,&argv,0,help);
PetscInt f = 2;
PetscTruth flg;
PetscLogStage assemblyStage, smaleStage;
char fileName[PETSC_MAX_PATH_LEN]="matlab/out.m";
PetscOptionsGetReal("-test_h", &h, PETSC_NULL);
PetscOptionsGetReal("-test_mi", &mi, PETSC_NULL);
PetscOptionsGetReal("-test_ro", &ro, PETSC_NULL);
PetscOptionsGetReal("-test_beta", &beta, PETSC_NULL);
PetscOptionsGetReal("-test_M", &M, PETSC_NULL);
PetscOptionsGetInt("-test_x", &xSubs, PETSC_NULL);
PetscOptionsGetInt("-test_y", &ySubs, PETSC_NULL);
PetscOptionsGetInt("-test_f", &f, PETSC_NULL);
PetscOptionsGetString(PETSC_NULL, "-test_out_file", fileName, PETSC_MAX_PATH_LEN-1, &flg);
//if (!flg) SETERRQ(1,"Must indicate binary file with the -test_out_file option");
{
//n = xSubs;
//l = ySubs;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
MPI_Comm_size(PETSC_COMM_WORLD, &size);
PetscViewer v;
PetscLogStageRegister("Assembly", &assemblyStage);
PetscLogStagePush(assemblyStage);
Mesh *mesh = new Mesh();
//mesh->generateRectangularMesh(m, n, k, l, h);
mesh->loadHDF5("mesh.med");
mesh->partition(size);
mesh->tear();
SDSystem sdSystem(mesh,fList[f],fList[0]);
PetscViewerBinaryOpen(PETSC_COMM_WORLD, "matlab/mesh.m", FILE_MODE_WRITE, &v);
mesh->dumpForMatlab(v);
mesh->saveHDF5("outmesh.med");
PetscViewerDestroy(v);
delete mesh;
PetscLogStagePop();
Smale smale(&sdSystem,mi,ro,beta,M);
PetscLogStageRegister("Smale", &smaleStage);
PetscLogStagePush(smaleStage);
smale.solve();
PetscLogStagePop();
saveScalarResultHDF5("outmesh.med","smale",smale.getx());
PetscViewerBinaryOpen(PETSC_COMM_WORLD, fileName, FILE_MODE_WRITE, &v);
smale.dumpSolution(v);
//smale.dumpSolution(matLabView);
PetscViewerDestroy(v);
//PetscViewerDestroy(matLabView);
}
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool interpolate = user->interpolate;
PetscReal refinementLimit = user->refinementLimit;
PetscBool cellSimplex = user->cellSimplex;
PetscBool cellWedge = user->cellWedge;
PetscBool simplex2tensor = user->simplex2tensor;
const char *filename = user->filename;
const char *bdfilename = user->bdfilename;
const char *extfilename = user->extfilename;
PetscBool testp4est_seq = user->testp4est[0];
PetscBool testp4est_par = user->testp4est[1];
PetscInt triSizes_n2[2] = {4, 4};
PetscInt triPoints_n2[8] = {3, 5, 6, 7, 0, 1, 2, 4};
PetscInt triSizes_n8[8] = {1, 1, 1, 1, 1, 1, 1, 1};
PetscInt triPoints_n8[8] = {0, 1, 2, 3, 4, 5, 6, 7};
PetscInt quadSizes[2] = {2, 2};
PetscInt quadPoints[4] = {2, 3, 0, 1};
PetscInt gmshSizes_n3[3] = {14, 14, 14};
PetscInt gmshPoints_n3[42] = {1, 2, 4, 5, 9, 10, 11, 15, 16, 20, 21, 27, 28, 29,
3, 8, 12, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
0, 6, 7, 13, 14, 17, 18, 19, 22, 23, 24, 25, 26, 41};
PetscInt fluentSizes_n3[3] = {50, 50, 50};
PetscInt fluentPoints_n3[150] = { 5, 6, 7, 8, 12, 14, 16, 34, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 48, 50, 51, 80, 81, 89,
91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 104, 121, 122, 124, 125, 126, 127, 128, 129, 131, 133, 143, 144, 145, 147,
1, 3, 4, 9, 10, 17, 18, 19, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 47, 61, 71, 72, 73, 74,
75, 76, 77, 78, 79, 86, 87, 88, 90, 92, 113, 115, 116, 117, 118, 119, 120, 123, 138, 140, 141, 142, 146, 148, 149,
0, 2, 11, 13, 15, 20, 21, 22, 23, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 67,
68, 69, 70, 82, 83, 84, 85, 102, 103, 105, 106, 107, 108, 109, 110, 111, 112, 114, 130, 132, 134, 135, 136, 137, 139};
size_t len, bdlen, extlen;
PetscMPIInt rank, size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(user->createMeshEvent,0,0,0,0);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
ierr = PetscStrlen(bdfilename, &bdlen);CHKERRQ(ierr);
ierr = PetscStrlen(extfilename, &extlen);CHKERRQ(ierr);
ierr = PetscLogStagePush(user->stages[STAGE_LOAD]);CHKERRQ(ierr);
if (len) {
ierr = DMPlexCreateFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
} else if (bdlen) {
DM boundary;
ierr = DMPlexCreateFromFile(comm, bdfilename, interpolate, &boundary);CHKERRQ(ierr);
ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
ierr = DMDestroy(&boundary);CHKERRQ(ierr);
} else if (extlen) {
DM edm;
ierr = DMPlexCreateFromFile(comm, extfilename, interpolate, &edm);CHKERRQ(ierr);
ierr = DMPlexExtrude(edm, user->extrude_layers, user->extrude_thickness, PETSC_TRUE, interpolate, dm);CHKERRQ(ierr);
ierr = DMDestroy(&edm);CHKERRQ(ierr);
} else {
switch (user->domainShape) {
case BOX:
if (cellWedge) {
if (dim != 3) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Dimension must be 3 for a wedge mesh, not %D", dim);
ierr = DMPlexCreateWedgeBoxMesh(comm, user->domainBoxSizes, user->domainBoxL, user->domainBoxU, user->periodicity, PETSC_FALSE, interpolate, dm);CHKERRQ(ierr);
} else {
ierr = DMPlexCreateBoxMesh(comm, dim, cellSimplex, user->domainBoxSizes, user->domainBoxL, user->domainBoxU, user->periodicity, interpolate, dm);CHKERRQ(ierr);
}
break;
case CYLINDER:
if (cellSimplex) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Cannot mesh a cylinder with simplices");
if (dim != 3) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Dimension must be 3 for a cylinder mesh, not %D", dim);
if (cellWedge) {
ierr = DMPlexCreateWedgeCylinderMesh(comm, 6, interpolate, dm);CHKERRQ(ierr);
} else {
ierr = DMPlexCreateHexCylinderMesh(comm, 3, user->periodicity[2], dm);CHKERRQ(ierr);
}
break;
default: SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Unknown domain shape %D", user->domainShape);
}
}
ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr); /* needed for periodic */
ierr = DMViewFromOptions(*dm,NULL,"-init_dm_view");CHKERRQ(ierr);
ierr = DMGetDimension(*dm,&dim);CHKERRQ(ierr);
if (testp4est_seq) {
#if defined(PETSC_HAVE_P4EST)
DM dmConv = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
user->cellSimplex = PETSC_FALSE;
ierr = DMConvert(*dm,dim == 2 ? DMP4EST : DMP8EST,&dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMSetFromOptions(dmConv);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-conv_seq_1_dm_view");CHKERRQ(ierr);
ierr = DMConvert(*dm,DMPLEX,&dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMViewFromOptions(*dm, NULL, "-conv_seq_2_dm_view");CHKERRQ(ierr);
#else
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Recompile with --download-p4est");
#endif
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (!testp4est_seq) {
DM refinedMesh = NULL;
DM distributedMesh = NULL;
if (user->testPartition) {
const PetscInt *sizes = NULL;
const PetscInt *points = NULL;
PetscPartitioner part;
if (!rank) {
if (dim == 2 && cellSimplex && size == 2) {
sizes = triSizes_n2; points = triPoints_n2;
} else if (dim == 2 && cellSimplex && size == 8) {
sizes = triSizes_n8; points = triPoints_n8;
} else if (dim == 2 && !cellSimplex && size == 2) {
sizes = quadSizes; points = quadPoints;
} else if (dim == 2 && size == 3) {
PetscInt Nc;
ierr = DMPlexGetHeightStratum(*dm, 0, NULL, &Nc);CHKERRQ(ierr);
if (Nc == 42) { /* Gmsh 3 & 4 */
sizes = gmshSizes_n3; points = gmshPoints_n3;
} else if (Nc == 150) { /* Fluent 1 */
sizes = fluentSizes_n3; points = fluentPoints_n3;
} else if (Nc == 42) { /* Med 1 */
} else if (Nc == 161) { /* Med 3 */
}
}
}
ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
ierr = PetscPartitionerSetType(part, PETSCPARTITIONERSHELL);CHKERRQ(ierr);
ierr = PetscPartitionerShellSetPartition(part, size, sizes, points);CHKERRQ(ierr);
} else {
PetscPartitioner part;
ierr = DMPlexGetPartitioner(*dm,&part);CHKERRQ(ierr);
ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
}
/* Distribute mesh over processes */
ierr = PetscLogStagePush(user->stages[STAGE_DISTRIBUTE]);CHKERRQ(ierr);
ierr = DMPlexDistribute(*dm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-distributed_dm_view");CHKERRQ(ierr);
/* Refine mesh using a volume constraint */
ierr = PetscLogStagePush(user->stages[STAGE_REFINE]);CHKERRQ(ierr);
ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
ierr = DMPlexSetRefinementLimit(*dm, refinementLimit);CHKERRQ(ierr);
ierr = DMRefine(*dm, comm, &refinedMesh);CHKERRQ(ierr);
if (refinedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = refinedMesh;
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
}
ierr = PetscLogStagePush(user->stages[STAGE_REFINE]);CHKERRQ(ierr);
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (testp4est_par) {
#if defined(PETSC_HAVE_P4EST)
DM dmConv = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
user->cellSimplex = PETSC_FALSE;
ierr = DMConvert(*dm,dim == 2 ? DMP4EST : DMP8EST,&dmConv);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) dmConv, "conv_par_1_");CHKERRQ(ierr);
if (dmConv) {
ierr = DMSetFromOptions(dmConv);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = DMConvert(*dm,DMPLEX,&dmConv);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) dmConv, "conv_par_2_");CHKERRQ(ierr);
if (dmConv) {
ierr = DMSetFromOptions(dmConv);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
#else
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Recompile with --download-p4est");
#endif
}
if (user->overlap) {
DM overlapMesh = NULL;
/* Add the level-1 overlap to refined mesh */
ierr = PetscLogStagePush(user->stages[STAGE_OVERLAP]);CHKERRQ(ierr);
ierr = DMPlexDistributeOverlap(*dm, 1, NULL, &overlapMesh);CHKERRQ(ierr);
if (overlapMesh) {
ierr = DMView(overlapMesh, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = overlapMesh;
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
}
if (simplex2tensor) {
DM rdm = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &rdm);CHKERRQ(ierr);
if (rdm) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = rdm;
}
user->cellSimplex = PETSC_FALSE;
}
ierr = PetscObjectSetName((PetscObject) *dm, "Simplicial Mesh");CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = PetscLogEventEnd(user->createMeshEvent,0,0,0,0);CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
int main(int argc,char **argv) {
PetscErrorCode ierr;
PetscBool view = PETSC_FALSE,
viewsoln = PETSC_FALSE,
noprealloc = PETSC_FALSE;
char root[256] = "", nodesname[256], issname[256], solnname[256];
UM mesh;
unfemCtx user;
SNES snes;
KSP ksp;
PC pc;
Mat A;
Vec r, u, uexact;
double err, h_max;
PetscInitialize(&argc,&argv,NULL,help);
ierr = PetscLogStageRegister("Read mesh ", &user.readstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Set-up ", &user.setupstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Solver ", &user.solverstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Residual eval ", &user.resstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Jacobian eval ", &user.jacstage); CHKERRQ(ierr); //STRIP
user.quaddeg = 1;
user.solncase = 0;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, "un_", "options for unfem", ""); CHKERRQ(ierr);
ierr = PetscOptionsInt("-case",
"exact solution cases: 0=linear, 1=nonlinear, 2=nonhomoNeumann, 3=chapter3, 4=koch",
"unfem.c",user.solncase,&(user.solncase),NULL); CHKERRQ(ierr);
ierr = PetscOptionsString("-mesh",
"file name root of mesh stored in PETSc binary with .vec,.is extensions",
"unfem.c",root,root,sizeof(root),NULL); CHKERRQ(ierr);
ierr = PetscOptionsInt("-quaddeg",
"quadrature degree (1,2,3)",
"unfem.c",user.quaddeg,&(user.quaddeg),NULL); CHKERRQ(ierr);
ierr = PetscOptionsBool("-view",
"view loaded nodes and elements at stdout",
"unfem.c",view,&view,NULL); CHKERRQ(ierr);
ierr = PetscOptionsBool("-view_solution",
"view solution u(x,y) to binary file; uses root name of mesh plus .soln\nsee petsc2tricontour.py to view graphically",
"unfem.c",viewsoln,&viewsoln,NULL); CHKERRQ(ierr);
ierr = PetscOptionsBool("-noprealloc",
"do not perform preallocation before matrix assembly",
"unfem.c",noprealloc,&noprealloc,NULL); CHKERRQ(ierr);
ierr = PetscOptionsEnd(); CHKERRQ(ierr);
// set parameters and exact solution
user.a_fcn = &a_lin;
user.f_fcn = &f_lin;
user.uexact_fcn = &uexact_lin;
user.gD_fcn = &gD_lin;
user.gN_fcn = &gN_lin;
switch (user.solncase) {
case 0 :
break;
case 1 :
user.a_fcn = &a_nonlin;
user.f_fcn = &f_nonlin;
break;
case 2 :
user.gN_fcn = &gN_linneu;
break;
case 3 :
user.a_fcn = &a_square;
user.f_fcn = &f_square;
user.uexact_fcn = &uexact_square;
user.gD_fcn = &gD_square;
user.gN_fcn = NULL; // seg fault if ever called
break;
case 4 :
user.a_fcn = &a_koch;
user.f_fcn = &f_koch;
user.uexact_fcn = NULL;
user.gD_fcn = &gD_koch;
user.gN_fcn = NULL; // seg fault if ever called
break;
default :
SETERRQ(PETSC_COMM_WORLD,1,"other solution cases not implemented");
}
// determine filenames
strcpy(nodesname, root);
strncat(nodesname, ".vec", 4);
strcpy(issname, root);
strncat(issname, ".is", 3);
//STARTMAININITIAL
PetscLogStagePush(user.readstage); //STRIP
// read mesh object of type UM
ierr = UMInitialize(&mesh); CHKERRQ(ierr);
ierr = UMReadNodes(&mesh,nodesname); CHKERRQ(ierr);
ierr = UMReadISs(&mesh,issname); CHKERRQ(ierr);
ierr = UMStats(&mesh, &h_max, NULL, NULL, NULL); CHKERRQ(ierr);
if (view) { //STRIP
PetscViewer stdoutviewer; //STRIP
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&stdoutviewer); CHKERRQ(ierr); //STRIP
ierr = UMViewASCII(&mesh,stdoutviewer); CHKERRQ(ierr); //STRIP
} //STRIP
user.mesh = &mesh;
PetscLogStagePop(); //STRIP
// configure Vecs and SNES
PetscLogStagePush(user.setupstage); //STRIP
ierr = VecCreate(PETSC_COMM_WORLD,&r); CHKERRQ(ierr);
ierr = VecSetSizes(r,PETSC_DECIDE,mesh.N); CHKERRQ(ierr);
ierr = VecSetFromOptions(r); CHKERRQ(ierr);
ierr = VecDuplicate(r,&u); CHKERRQ(ierr);
ierr = VecSet(u,0.0); CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes); CHKERRQ(ierr);
ierr = SNESSetFunction(snes,r,FormFunction,&user); CHKERRQ(ierr);
// reset default KSP and PC
ierr = SNESGetKSP(snes,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG); CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCICC); CHKERRQ(ierr);
// setup matrix for Picard iteration, including preallocation
ierr = MatCreate(PETSC_COMM_WORLD,&A); CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,mesh.N,mesh.N); CHKERRQ(ierr);
ierr = MatSetFromOptions(A); CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE); CHKERRQ(ierr);
if (noprealloc) {
ierr = MatSetUp(A); CHKERRQ(ierr);
} else {
ierr = Preallocation(A,&user); CHKERRQ(ierr);
}
ierr = SNESSetJacobian(snes,A,A,FormPicard,&user); CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes); CHKERRQ(ierr);
PetscLogStagePop(); //STRIP
// solve
PetscLogStagePush(user.solverstage); //STRIP
ierr = SNESSolve(snes,NULL,u);CHKERRQ(ierr);
PetscLogStagePop(); //STRIP
//ENDMAININITIAL
if (viewsoln) {
strcpy(solnname, root);
strncat(solnname, ".soln", 5);
ierr = UMViewSolutionBinary(&mesh,solnname,u); CHKERRQ(ierr);
}
if (user.uexact_fcn) {
// measure error relative to exact solution
ierr = VecDuplicate(r,&uexact); CHKERRQ(ierr);
ierr = FillExact(uexact,&user); CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,uexact); CHKERRQ(ierr); // u <- u + (-1.0) uexact
ierr = VecNorm(u,NORM_INFINITY,&err); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"case %d result for N=%d nodes with h = %.3e : |u-u_ex|_inf = %g\n",
user.solncase,mesh.N,h_max,err); CHKERRQ(ierr);
VecDestroy(&uexact);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
"case %d completed for N=%d nodes with h = %.3e (no exact solution)\n",
user.solncase,mesh.N,h_max); CHKERRQ(ierr);
}
// clean-up
SNESDestroy(&snes);
MatDestroy(&A);
VecDestroy(&u); VecDestroy(&r);
UMDestroy(&mesh);
PetscFinalize();
return 0;
}