PetscErrorCode SetInitialCondition(DM dm, Vec X, User user)
{
DM dmCell;
const PetscScalar *cellgeom;
PetscScalar *x;
PetscInt cStart, cEnd, cEndInterior = user->cEndInterior, c;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = VecGetDM(user->cellgeom, &dmCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(X, &x);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
const CellGeom *cg;
PetscScalar *xc;
ierr = DMPlexPointLocalRead(dmCell,c,cellgeom,&cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm,c,x,&xc);CHKERRQ(ierr);
if (xc) {
ierr = InitialCondition(0.0, cg->centroid, xc, user);CHKERRQ(ierr);
}
}
ierr = VecRestoreArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode SetInitialCondition(DM dm, Vec X, User user)
{
DM dmCell;
const PetscScalar *cellgeom;
PetscScalar *x;
PetscInt cStart, cEnd, cEndInterior = user->cEndInterior, c;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = VecGetDM(user->cellgeom, &dmCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(X, &x);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
const CellGeom *cg;
PetscScalar *xc;
ierr = DMPlexPointLocalRead(dmCell,c,cellgeom,&cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm,c,x,&xc);CHKERRQ(ierr);
if (xc) {
ierr = InitialCondition(0.0, cg->centroid, xc, user);CHKERRQ(ierr);
}
}
ierr = VecRestoreArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);
{ //Apply the Boundary condition for the intial condition
Vec XLocal;
ierr = DMGetLocalVector(user->dm, &XLocal);CHKERRQ(ierr);
ierr = VecSet(XLocal, 0);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(user->dm, X, INSERT_VALUES, XLocal);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->dm, X, INSERT_VALUES, XLocal);CHKERRQ(ierr);
ierr = ApplyBC(user->dm, user->current_time, XLocal, user);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(user->dm, XLocal, INSERT_VALUES, X);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(user->dm, XLocal, INSERT_VALUES, X);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
TS ts; //Time stepper
Vec soln; //Holds the solution vector, including all the primitive
//variables.
DM dmda; //Manages the computational grid and parallelization.
int X1Start, X2Start;
int X1Size, X2Size;
PetscInitialize(&argc, &argv, PETSC_NULL, help);
// Create the computational domain.
DMDACreate2d(PETSC_COMM_WORLD,
DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED,
DMDA_STENCIL_STAR,
N1, N2,
PETSC_DECIDE, PETSC_DECIDE,
DOF, NG, PETSC_NULL, PETSC_NULL, &dmda);
// When running in parallel, each process computes from
// [X1Start, X1Start+X1Size] x [X2Start, X2Start+X2Size]
DMDAGetCorners(dmda,
&X1Start, &X2Start, NULL,
&X1Size, &X2Size, NULL);
// Create the solution vector.
DMCreateGlobalVector(dmda, &soln);
// Create the time stepper and link it to the computational grid and the
// residual evaluation function.
TSCreate(PETSC_COMM_WORLD, &ts);
TSSetDM(ts, dmda);
TSSetIFunction(ts, PETSC_NULL, ComputeResidual, NULL);
// OpenCL boilerplate code.
clErr = cl::Platform::get(&platforms);
CheckCLErrors(clErr, "cl::Platform::get");
// Select computation device here.
clErr = platforms.at(1).getDevices(CL_DEVICE_TYPE_CPU, &devices);
CheckCLErrors(clErr, "cl::Platform::getDevices");
context = cl::Context(devices, NULL, NULL, NULL, &clErr);
CheckCLErrors(clErr, "cl::Context::Context");
queue = cl::CommandQueue(context, devices.at(0), 0, &clErr);
CheckCLErrors(clErr, "cl::CommandQueue::CommandQueue");
std::ifstream sourceFile("computeresidual.cl");
std::string sourceCode((std::istreambuf_iterator<char>(sourceFile)),
std::istreambuf_iterator<char>());
cl::Program::Sources source(1, std::make_pair(sourceCode.c_str(),
sourceCode.length()+1));
program = cl::Program(context, source, &clErr);
CheckCLErrors(clErr, "cl::Program::Program");
// Pass in constants to the OpenCL kernel as compiler switches. This is an
// efficient way to handle constants such as domain sizes in OpenCL.
std::string BuildOptions("\
-D X1_SIZE=" +
std::to_string(X1Size) +
" -D X2_SIZE=" +
std::to_string(X2Size) +
" -D TOTAL_X1_SIZE=" +
std::to_string(X1Size+2*NG) +
" -D TOTAL_X2_SIZE=" +
std::to_string(X2Size+2*NG));
// Compile the OpenCL program and extract the kernel.
PetscScalar start = std::clock();
clErr = program.build(devices, BuildOptions.c_str(), NULL, NULL);
const char *buildlog = program.getBuildInfo<CL_PROGRAM_BUILD_LOG>(
devices.at(0),
&clErr).c_str();
PetscPrintf(PETSC_COMM_WORLD, "%s\n", buildlog);
CheckCLErrors(clErr, "cl::Program::build");
PetscScalar end = std::clock();
PetscScalar time = (end - start)/(PetscScalar)CLOCKS_PER_SEC;
PetscPrintf(PETSC_COMM_WORLD,
"Time taken for kernel compilation = %f\n", time);
kernel = cl::Kernel(program, "ComputeResidual", &clErr);
CheckCLErrors(clErr, "cl::Kernel::Kernel");
// How much memory is the kernel using?
cl_ulong localMemSize = kernel.getWorkGroupInfo<CL_KERNEL_LOCAL_MEM_SIZE>(
devices.at(0), &clErr);
cl_ulong privateMemSize = kernel.getWorkGroupInfo<CL_KERNEL_PRIVATE_MEM_SIZE>(
devices.at(0), &clErr);
printf("Local memory used = %llu\n", (unsigned long long)localMemSize);
printf("Private memory used = %llu\n", (unsigned long long)privateMemSize);
// Set initial conditions.
InitialCondition(ts, soln);
TSSetSolution(ts, soln);
TSSetType(ts, TSTHETA);
TSSetFromOptions(ts);
// Finally solve! All time stepping options can be controlled from the
// command line.
TSSolve(ts, soln);
// Delete the data structures in the following order.
DMDestroy(&dmda);
VecDestroy(&soln);
TSDestroy(&ts);
PetscFinalize();
return(0);
}
