PetscErrorCode
Metos3DBGCDomainConditionFinal(Metos3D *metos3d)
{
// work vars
PetscInt ndc = metos3d->domainConditionCount;
PetscFunctionBegin;
if (ndc > 0) {
// work vars
PetscInt idc;
PetscInt *ndcs = metos3d->domainConditionCountArray;
Vec **dcs = metos3d->domainConditionArray;
// loop
for (idc = 0; idc < ndc; idc++) {
// vec array
VecDestroyVecs(ndcs[idc], &dcs[idc]);
}
PetscFree(dcs);
PetscFree(ndcs);
// bgc api
VecDestroyVecs(ndc, &metos3d->bgcdc);
VecDestroyVecs(1, &metos3d->bgcdcBD);
}
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCDomainConditionFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCBoundaryConditionFinal(Metos3D *metos3d)
{
// work vars
PetscInt nbc = metos3d->boundaryConditionCount;
PetscFunctionBegin;
if (nbc > 0) {
// work vars
PetscInt ibc;
PetscInt *nbcs = metos3d->boundaryConditionCountArray;
Vec **bcs = metos3d->boundaryConditionArray;
// loop
for (ibc = 0; ibc < nbc; ibc++) {
// vec array
VecDestroyVecs(nbcs[ibc], &bcs[ibc]);
}
// count array
// value array
PetscFree(bcs);
PetscFree(nbcs);
// bgc api
VecDestroyVecs(nbc, &metos3d->bgcbc);
VecDestroyVecs(1, &metos3d->bgcbcBD);
}
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCBoundaryConditionFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DTimeStepFinal(Metos3D *metos3d)
{
PetscFunctionBegin;
// debug stop
Metos3DDebug(metos3d, kDebugLevel, "Metos3DTimeStepFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DLoadFinal(Metos3D *metos3d)
{
PetscFunctionBegin;
// local indices
PetscFree(metos3d->profileStartLocal);
PetscFree(metos3d->profileEndLocal);
// debug stop
Metos3DDebug(metos3d, kDebugLevel, "Metos3DLoadFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCParameterFinal(Metos3D *metos3d)
{
// parameter
PetscInt nparam = metos3d->parameterCount;
PetscFunctionBegin;
if (nparam > 0) {
PetscFree(metos3d->u0);
}
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCParameterFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCFinal(Metos3D *metos3d)
{
PetscFunctionBegin;
// final parameter, domain, boundary, tracer
Metos3DBGCDomainConditionFinal(metos3d);
Metos3DBGCBoundaryConditionFinal(metos3d);
Metos3DBGCParameterFinal(metos3d);
Metos3DBGCTracerFinal(metos3d);
// debug stop
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DTimeStepInit(Metos3D *metos3d)
{
PetscFunctionBegin;
// register event
PetscLogEventRegister("TimeStepPhi", 0, &metos3d->eventTimeStepPhi);
// options
Metos3DUtilOptionsGetScalar(metos3d, "-Metos3DTimeStep", &metos3d->timeStep);
Metos3DUtilOptionsGetScalar(metos3d, "-Metos3DTimeStepStart", &metos3d->timeStepStart);
Metos3DUtilOptionsGetInt(metos3d, "-Metos3DTimeStepCount", &metos3d->timeStepCount);
// debug stop
Metos3DDebug(metos3d, kDebugLevel, "Metos3DTimeStepInit\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCTracerFinal(Metos3D *metos3d)
{
// bgc
PetscInt ntracer = metos3d->tracerCount;
PetscFunctionBegin;
// initial value vector
VecDestroyVecs(ntracer, &metos3d->y0);
// work vectors
VecDestroyVecs(1, &metos3d->ybgcinBD);
VecDestroyVecs(1, &metos3d->ybgcoutBD);
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCTracerFinal\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCInit(Metos3D *metos3d)
{
PetscFunctionBegin;
// register event
PetscLogEventRegister("BGCStep", 0, &metos3d->eventBGCStep);
// init tracer, boundary, domain, parameter
Metos3DBGCTracerInit(metos3d);
Metos3DBGCParameterInit(metos3d);
Metos3DBGCBoundaryConditionInit(metos3d);
Metos3DBGCDomainConditionInit(metos3d);
// debug stop
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCInit\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DTimeStepPhiStep(Metos3D *metos3d, PetscScalar t, PetscScalar dt, PetscInt istep, Vec *yin, Vec *yout, Vec *ywork, PetscInt nparam, PetscReal *u0)
{
// bgc
PetscInt ntracer = metos3d->tracerCount;
// transport
PetscInt nmat = metos3d->matrixCount;
Mat *Ae = metos3d->matrixExplicitArray;
Mat *Ai = metos3d->matrixImplicitArray;
Mat *Aework = &metos3d->Aework;
Mat *Aiwork = &metos3d->Aiwork;
// work vars
PetscInt itracer;
PetscFunctionBegin;
// bgc
Metos3DBGCStep(metos3d, t, dt, yin, yout, nparam, u0);
char filePrefixFormat[PETSC_MAX_PATH_LEN];
char filePrefix [PETSC_MAX_PATH_LEN];
if ((metos3d->spinupStep + 1)%metos3d->moduloStep[0] == 0) {
PetscInt modstep = metos3d->moduloStepCount;
if (modstep > 0) {
PetscInt imodstep = metos3d->moduloStep[1];
if ((istep+1)%imodstep == 0) {
sprintf(filePrefixFormat, "bgc/q_%s%s", metos3d->filePrefix, metos3d->fileFormatPrefix[1]);
sprintf(filePrefix, filePrefixFormat, istep);
// output
Metos3DBGCOutputPrefix(metos3d, filePrefix, ntracer, yout);
}
} else {
sprintf(filePrefixFormat, "bgc/q_%s%s", metos3d->filePrefix, metos3d->fileFormatPrefix[1]);
sprintf(filePrefix, filePrefixFormat, istep);
// output
Metos3DBGCOutputPrefix(metos3d, filePrefix, ntracer, yout);
}
}
// transport
// ywork = Ae*yin
// ywork = ywork + yout
// yout = Ai*ywork
Metos3DTransport(metos3d, t, nmat, Ae, ntracer, yin, ywork, Aework);
for (itracer = 0; itracer < ntracer; itracer++) VecAXPY(ywork[itracer], 1.0, yout[itracer]);
Metos3DTransport(metos3d, t, nmat, Ai, ntracer, ywork, yout, Aiwork);
// debug
Metos3DDebug(metos3d, kDebugLevel, FSSDSE, "Metos3DTimeStepPhiStep", "istep:", istep, "t:", t);
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCParameterInit(Metos3D *metos3d)
{
// work vars
PetscInt nparam;
PetscFunctionBegin;
// parameter count
Metos3DUtilOptionsGetInt(metos3d, "-Metos3DParameterCount", &nparam);
metos3d->parameterCount = nparam;
if (nparam > 0) {
// work vars
PetscInt nmax;
// real array
nmax = nparam;
PetscMalloc(nmax*sizeof(PetscReal),&metos3d->u0);
Metos3DUtilOptionsGetRealArray(metos3d, "-Metos3DParameterValue", &nmax, metos3d->u0);
}
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCParameterInit\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCOutputPrefix(Metos3D *metos3d, char *prefix, PetscInt n, Vec *v)
{
// work vars
PetscInt nmax, i_tracer;
PetscBool flag = PETSC_FALSE;
char outputDirectory [PETSC_MAX_PATH_LEN];
char *outputFileNames [PETSC_MAX_PATH_LEN];
char outputFileNameFormat[PETSC_MAX_PATH_LEN];
char format [PETSC_MAX_PATH_LEN];
char filePath [PETSC_MAX_PATH_LEN];
PetscFunctionBegin;
// output tracer vectors to disk
Metos3DUtilOptionsGetString(metos3d, "-Metos3DTracerOutputDirectory", outputDirectory);
PetscOptionsGetString(PETSC_NULL, "-Metos3DTracerOutputFileFormat", outputFileNameFormat, PETSC_MAX_PATH_LEN, &flag);
if (flag == PETSC_TRUE) {
Metos3DUtilFormatParse(metos3d, outputFileNameFormat);
for (i_tracer = 0; i_tracer < n; i_tracer++)
{
sprintf(format, "%s%s%s", outputDirectory, prefix, outputFileNameFormat);
sprintf(filePath, format, i_tracer);
Metos3DUtilVectorView(metos3d, filePath, &v[i_tracer]);
}
} else {
nmax = n;
PetscOptionsGetStringArray(PETSC_NULL, "-Metos3DTracerOutputFile", outputFileNames, &nmax, &flag);
if (flag == PETSC_TRUE) {
for (i_tracer = 0; i_tracer < n; i_tracer++)
{
sprintf(filePath, "%s%s%s", outputDirectory, prefix, outputFileNames[i_tracer]);
Metos3DUtilVectorView(metos3d, filePath, &v[i_tracer]);
PetscFree(outputFileNames[i_tracer]);
}
}
}
// debug
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCOutputPrefix\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DTimeStepFunction(SNES snes, Vec ynBD, Vec fnBD, void *ctx)
{
Metos3D *metos3d = (Metos3D*)ctx;
// geometry
PetscInt nvec = metos3d->vectorLength;
// bgc
PetscInt ntracer = metos3d->tracerCount;
// load
PetscInt nvecloc = metos3d->vectorLengthLocal;
// parameter
PetscInt nparam = metos3d->parameterCount;
PetscReal *u0 = metos3d->u0;
// work vars
PetscInt itracer;
Vec *yin, *yinold, *yout;
PetscFunctionBegin;
// create work vectors
Metos3DUtilVecCreateAndSetValue(metos3d, ntracer, nvec, nvecloc, &yin, 0.0);
Metos3DUtilVecCreateAndSetValue(metos3d, ntracer, nvec, nvecloc, &yinold, 0.0);
Metos3DUtilVecCreateAndSetValue(metos3d, ntracer, nvec, nvecloc, &yout, 0.0);
// yin = ynBD
// yinold = ynBD
// yout = Phi(yin)
// yout = - yout + yinold;
// fnBD = yout
Metos3DUtilVecCopyDiagonalToSeparate(metos3d, ntracer, nvecloc, &ynBD, yin);
Metos3DUtilVecCopyDiagonalToSeparate(metos3d, ntracer, nvecloc, &ynBD, yinold);
Metos3DTimeStepPhi(metos3d, yin, yout, nparam, u0);
for (itracer = 0; itracer < ntracer; itracer++) VecAYPX(yout[itracer], -1.0, yinold[itracer]);
Metos3DUtilVecCopySeparateToDiagonal(metos3d, ntracer, nvecloc, yout, &fnBD);
// free work vectors
VecDestroyVecs(ntracer, &yin);
VecDestroyVecs(ntracer, &yinold);
VecDestroyVecs(ntracer, &yout);
// debug
Metos3DDebug(metos3d, kDebugLevel, "Metos3DTimeStepFunction\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCStepDomainCondition(Metos3D *metos3d, PetscScalar t)
{
// bgc
PetscInt ndc = metos3d->domainConditionCount;
PetscInt *ndcs = metos3d->domainConditionCountArray;
Vec **dcs = metos3d->domainConditionArray;
Vec *bgcdc = metos3d->bgcdc;
// work vars
PetscInt idc;
PetscFunctionBegin;
// check count
if (ndc > 0) {
// prepare boundary condition
for (idc = 0; idc < ndc; idc++) {
// work vars
PetscInt ialpha, ibeta;
PetscScalar alpha, beta;
// interpolate
Metos3DUtilInterpolate(metos3d, t, ndcs[idc], &ialpha, &alpha, &ibeta, &beta);
// set
// bgcdc[idc] = dcs[idc][ialpha]
// bgcdc[idc] = alpha*bgcdc[idc]
// bgcdc[idc] = bgcdc[idc] + beta*dcs[idc][ibeta]
VecCopy (dcs[idc][ialpha], bgcdc[idc]);
VecScale(bgcdc[idc], alpha);
VecAXPY(bgcdc[idc], beta, dcs[idc][ibeta]);
}
// copy to one vector
PetscInt nvecloc = metos3d->vectorLengthLocal;
Metos3DUtilVecCopySeparateToDiagonal(metos3d, ndc, nvecloc, bgcdc, metos3d->bgcdcBD);
}
// debug
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCStepDomainCondition\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCStepBoundaryCondition(Metos3D *metos3d, PetscScalar t)
{
// bgc
PetscInt nbc = metos3d->boundaryConditionCount;
PetscInt *nbcs = metos3d->boundaryConditionCountArray;
Vec **bcs = metos3d->boundaryConditionArray;
Vec *bgcbc = metos3d->bgcbc;
// work vars
PetscInt ibc;
PetscFunctionBegin;
// check count
if (nbc > 0) {
// prepare boundary condition
for (ibc = 0; ibc < nbc; ibc++) {
// work vars
PetscInt ialpha, ibeta;
PetscScalar alpha, beta;
// interpolate
Metos3DUtilInterpolate(metos3d, t, nbcs[ibc], &ialpha, &alpha, &ibeta, &beta);
// set
// bgcbc[ibc] = bcs[ibc][ialpha]
// bgcbc[ibc] = alpha*bgcbc[ibc]
// bgcbc[ibc] = bgcbc[ibc] + beta*bcs[ibc][ibeta]
VecCopy (bcs[ibc][ialpha], bgcbc[ibc]);
VecScale(bgcbc[ibc], alpha);
VecAXPY(bgcbc[ibc], beta, bcs[ibc][ibeta]);
}
// copy to one vector
PetscInt nprofloc = metos3d->profileCountLocal;
Metos3DUtilVecCopySeparateToDiagonalProfile(metos3d, nbc, nprofloc, bgcbc, metos3d->bgcbcBD);
}
// debug
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCStepBoundaryCondition\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCTracerInit(Metos3D *metos3d)
{
// load
PetscInt nvec = metos3d->vectorLength;
PetscInt nvecloc = metos3d->vectorLengthLocal;
// work vars
PetscInt ntracer, itracer;
PetscInt nmax;
PetscBool flag;
char initFileNameFormat [PETSC_MAX_PATH_LEN];
PetscFunctionBegin;
// tracer count
Metos3DUtilOptionsGetInt(metos3d, "-Metos3DTracerCount", &ntracer);
metos3d->tracerCount = ntracer;
// read tracer init options
// order:
// 1. file name format
// 2. file names
// 3. constant values
PetscOptionsGetString(PETSC_NULL, "-Metos3DTracerInitFileFormat", initFileNameFormat, PETSC_MAX_PATH_LEN, &flag);
if (flag == PETSC_TRUE) {
// work vars
char tracerInputDirectory[PETSC_MAX_PATH_LEN];
char format [PETSC_MAX_PATH_LEN];
char filePath [PETSC_MAX_PATH_LEN];
// input directory, vector, format
Metos3DUtilOptionsGetString(metos3d, "-Metos3DTracerInputDirectory", tracerInputDirectory);
Metos3DUtilVecCreateAndSetValue(metos3d, ntracer, nvec, nvecloc, &metos3d->y0, 0.0);
Metos3DUtilFormatParse(metos3d, initFileNameFormat);
for (itracer = 0; itracer < ntracer; itracer++)
{
sprintf(format, "%s%s", tracerInputDirectory, initFileNameFormat);
sprintf(filePath, format, itracer);
Metos3DUtilVectorLoad(metos3d, filePath, &metos3d->y0[itracer]);
}
} else {
// work vars
char tracerInputDirectory[PETSC_MAX_PATH_LEN];
char *initFileNames [PETSC_MAX_PATH_LEN];
char filePath [PETSC_MAX_PATH_LEN];
// file name
nmax = ntracer;
PetscOptionsGetStringArray(PETSC_NULL, "-Metos3DTracerInitFile", initFileNames, &nmax, &flag);
if (flag == PETSC_TRUE)
{
// input directory, vector, file name
Metos3DUtilOptionsGetString(metos3d, "-Metos3DTracerInputDirectory", tracerInputDirectory);
Metos3DUtilVecCreateAndSetValue(metos3d, ntracer, nvec, nvecloc, &metos3d->y0, 0.0);
for (itracer = 0; itracer < ntracer; itracer++)
{
sprintf(filePath, "%s%s", tracerInputDirectory, initFileNames[itracer]);
Metos3DUtilVectorLoad(metos3d, filePath, &metos3d->y0[itracer]);
PetscFree(initFileNames[itracer]);
}
}
else
{
// work vars
PetscReal *y0value;
// value
nmax = ntracer;
PetscMalloc(nmax*sizeof(PetscReal),&y0value);
// array, vector
Metos3DUtilOptionsGetRealArray(metos3d, "-Metos3DTracerInitValue", &nmax, y0value);
Metos3DUtilVecCreateAndSetValues(metos3d, ntracer, nvec, nvecloc, &metos3d->y0, y0value);
PetscFree(y0value);
}
}
// create work vectors
Metos3DUtilVecCreateAndSetValue(metos3d, 1, ntracer*nvec, ntracer*nvecloc, &metos3d->ybgcinBD, 0.0);
Metos3DUtilVecCreateAndSetValue(metos3d, 1, ntracer*nvec, ntracer*nvecloc, &metos3d->ybgcoutBD, 0.0);
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCTracerInit\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DLoadInit(Metos3D *metos3d)
{
MPI_Comm comm = metos3d->comm;
PetscInt debug = metos3d->debugLevel;
// geometry
PetscInt nprof = metos3d->profileCount;
PetscInt nvec = metos3d->vectorLength;
PetscInt *istart = metos3d->profileStart;
PetscInt *iend = metos3d->profileEnd;
// work vars
PetscInt nproc, myproc, iproc, iprof;
PetscInt *sumvecloc, *sumvecprev, *sumprofloc, *sumprofprev;
PetscInt nvecloc, nvecprev, nprofloc, nprofprev;
PetscInt *istartloc, *iendloc;
PetscFunctionBegin;
// determine process count and my process number
MPI_Comm_size(comm, &nproc);
MPI_Comm_rank(comm, &myproc);
// store
metos3d->processCount = nproc;
metos3d->processMine = myproc;
Metos3DDebug(metos3d, kDebugLevel3, F2SD, "Metos3DLoadInit", "processCount:", nproc);
// compute simple load distribution
PetscMalloc(nproc*sizeof(PetscInt), &sumvecloc);
PetscMalloc(nproc*sizeof(PetscInt), &sumvecprev);
PetscMalloc(nproc*sizeof(PetscInt), &sumprofloc);
PetscMalloc(nproc*sizeof(PetscInt), &sumprofprev);
//
// new algorithm, load balancing
//
// zero memory
PetscMemzero(sumvecloc, nproc*sizeof(PetscInt));
PetscMemzero(sumprofloc, nproc*sizeof(PetscInt));
iproc = 0;
for (iprof = 0; iprof < nprof; iprof++)
{
// compute index
iproc = (PetscInt)floor(((istart[iprof]-1.0+0.5*(iend[iprof]-istart[iprof]+1.0))/nvec)*nproc);
// sum profiles and vector elements per process
sumvecloc [iproc] = sumvecloc [iproc] + iend[iprof] - istart[iprof] + 1;
sumprofloc[iproc] = sumprofloc[iproc] + 1;
}
// //
// // old algorithm, no load balancing
// //
// // zero memory
// PetscMemzero(sumvecloc, nproc*sizeof(PetscInt));
// PetscMemzero(sumprofloc, nproc*sizeof(PetscInt));
// // compute local first
// PetscInt nvecopt;
// nvecopt = nvec/nproc;
// iproc = 0;
// for (iprof = 0; iprof < nprof; iprof++)
// {
// // sum profiles and vector elements per process
// sumvecloc [iproc] = sumvecloc [iproc] + iend[iprof] - istart[iprof] + 1;
// sumprofloc[iproc] = sumprofloc[iproc] + 1;
// if (sumvecloc[iproc] >= nvecopt) iproc++;
// }
// compute previous
PetscMemzero(sumvecprev, nproc*sizeof(PetscInt));
PetscMemzero(sumprofprev, nproc*sizeof(PetscInt));
for (iproc = 1; iproc < nproc; iproc++)
{
sumvecprev [iproc] = sumvecprev [iproc-1] + sumvecloc [iproc-1];
sumprofprev[iproc] = sumprofprev[iproc-1] + sumprofloc[iproc-1];
}
// compute local indices (1 indexed, same as global)
nvecloc = sumvecloc[myproc];
nprofloc = sumprofloc[myproc];
nvecprev = sumvecprev[myproc];
nprofprev = sumprofprev[myproc];
// free sums
PetscFree(sumvecloc);
PetscFree(sumvecprev);
PetscFree(sumprofloc);
PetscFree(sumprofprev);
// alloc memory
PetscMalloc(nprofloc*sizeof(PetscInt), &istartloc);
PetscMalloc(nprofloc*sizeof(PetscInt), &iendloc);
for (iprof = 0; iprof < nprofloc; iprof++)
{
istartloc[iprof] = istart[iprof+nprofprev] - nvecprev;
iendloc [iprof] = iend [iprof+nprofprev] - nvecprev;
}
// store
metos3d->vectorLengthLocal = nvecloc;
metos3d->vectorLengthPrevious = nvecprev;
metos3d->profileCountLocal = nprofloc;
metos3d->profileCountPrevious = nprofprev;
metos3d->profileStartLocal = istartloc;
metos3d->profileEndLocal = iendloc;
// debug
Metos3DSynchronizedDebug11(debug, kDebugLevel3, comm, SYNCFS5SD, "Metos3DLoadInit", "myproc:", myproc,
"nvecloc:", nvecloc, "nvecprev:", nvecprev, "nprofloc:", nprofloc, "nprofprev:", nprofprev);
PetscSynchronizedFlush(comm, PETSC_STDOUT);
// debug stop
Metos3DDebug(metos3d, kDebugLevel, "Metos3DLoadInit\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCDomainConditionInit(Metos3D *metos3d)
{
// work vars
PetscInt ndc;
PetscFunctionBegin;
// count
Metos3DUtilOptionsGetInt(metos3d, "-Metos3DDomainConditionCount", &ndc);
metos3d->domainConditionCount = ndc;
if (ndc > 0) {
// geometry
PetscInt nvec = metos3d->vectorLength;
// load
PetscInt nvecloc = metos3d->vectorLengthLocal;
// work vars
PetscInt *ndcs;
char inputDirectory [PETSC_MAX_PATH_LEN];
char *conditionName [PETSC_MAX_PATH_LEN];
char message [PETSC_MAX_PATH_LEN];
char optionName [PETSC_MAX_PATH_LEN];
char fileFormat [PETSC_MAX_PATH_LEN];
char filePath [PETSC_MAX_PATH_LEN];
char fileName [PETSC_MAX_PATH_LEN];
PetscInt nmax, idc, idcs;
PetscBool flag;
// count array
PetscMalloc(ndc*sizeof(PetscInt),&ndcs);
metos3d->domainConditionCountArray = ndcs;
// input directory
Metos3DUtilOptionsGetString(metos3d, "-Metos3DDomainConditionInputDirectory", inputDirectory);
// name
nmax = ndc;
PetscOptionsGetStringArray(PETSC_NULL, "-Metos3DDomainConditionName", conditionName, &nmax, &flag);
sprintf(message, "Please provide the '%s' option", "-Metos3DDomainConditionName");
Metos3DFlag(flag, message);
//
PetscMalloc(ndc*sizeof(Vec*), &metos3d->domainConditionArray);
// condition
for (idc = 0; idc < ndc; idc++) {
Metos3DDebug(metos3d, kDebugLevel, F3S, "Metos3DBGCDomainConditionInit", "name:", conditionName[idc]);
// count
sprintf(optionName, "-Metos3D%sCount", conditionName[idc]);
Metos3DUtilOptionsGetInt(metos3d, optionName, &ndcs[idc]);
// name
sprintf(optionName, "-Metos3D%sFileFormat", conditionName[idc]);
Metos3DUtilOptionsGetString(metos3d, optionName, fileFormat);
Metos3DUtilFormatParse(metos3d, fileFormat);
// create vector
Metos3DUtilVecCreateAndSetValue(metos3d, ndcs[idc], nvec, nvecloc, &metos3d->domainConditionArray[idc], 0.0);
// files
for (idcs = 0; idcs < ndcs[idc]; idcs++) {
// file path
sprintf(fileName, fileFormat, idcs);
sprintf(filePath, "%s%s", inputDirectory, fileName);
// load
Metos3DUtilVectorLoad(metos3d, filePath, &metos3d->domainConditionArray[idc][idcs]);
}
// name
PetscFree(conditionName[idc]);
}
// bgc api
Metos3DUtilVecCreateAndSetValue(metos3d, ndc, nvec, nvecloc, &metos3d->bgcdc, 0.0);
Metos3DUtilVecCreateAndSetValue(metos3d, 1, ndc*nvec, ndc*nvecloc, &metos3d->bgcdcBD, 0.0);
}
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCDomainConditionInit\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DTimeStepPhi(Metos3D *metos3d, Vec *yin, Vec *yout, PetscInt nparam, PetscReal *u0)
{
// bgc
PetscInt npref = metos3d->fileFormatPrefixCount;
PetscInt ntracer = metos3d->tracerCount;
// time step
PetscScalar t0 = metos3d->timeStepStart;
PetscScalar dt = metos3d->timeStep;
PetscInt nstep = metos3d->timeStepCount;
// work vars
PetscScalar tj;
PetscInt itracer, istep;
Vec *ywork;
PetscFunctionBegin;
// wait for all processors
PetscBarrier(PETSC_NULL);
// start log event
PetscLogEventBegin(metos3d->eventTimeStepPhi, 0, 0, 0, 0);
// prepare work vector
VecDuplicateVecs(*yin, ntracer, &ywork);
// initial point in time, project time to [0,1[
tj = fmod(t0, 1.0);
// init bgc, yout not set (yet)
Metos3DBGCStepInit(metos3d, tj, dt, yin, yout, nparam, u0);
// time step loop
for (istep = 0; istep < nstep; istep++) {
// point in time
tj = fmod(t0 + istep*dt, 1.0);
// work vars
char filePrefixFormat[PETSC_MAX_PATH_LEN];
char filePrefix [PETSC_MAX_PATH_LEN];
// file prefix
if (npref > 1) {
if ((metos3d->spinupStep + 1)%metos3d->moduloStep[0] == 0) {
PetscInt modstep = metos3d->moduloStepCount;
if (modstep > 0) {
PetscInt imodstep = metos3d->moduloStep[1];
if ((istep+1)%imodstep == 0) {
sprintf(filePrefixFormat, "%s%s", metos3d->filePrefix, metos3d->fileFormatPrefix[1]);
sprintf(filePrefix, filePrefixFormat, istep);
// output
Metos3DBGCOutputPrefix(metos3d, filePrefix, ntracer, yin);
}
} else {
sprintf(filePrefixFormat, "%s%s", metos3d->filePrefix, metos3d->fileFormatPrefix[1]);
sprintf(filePrefix, filePrefixFormat, istep);
// output
Metos3DBGCOutputPrefix(metos3d, filePrefix, ntracer, yin);
}
}
}
// yout = Phi(yi)
// yin = yout
Metos3DTimeStepPhiStep(metos3d, tj, dt, istep, yin, yout, ywork, nparam, u0);
for(itracer = 0; itracer < ntracer; itracer++) VecCopy(yout[itracer], yin[itracer]);
}
// final bgc, yout not set (yet)
Metos3DBGCStepFinal(metos3d, tj, dt, yin, yout, nparam, u0);
// free work vector
VecDestroyVecs(ntracer, &ywork);
// wait for all processors
PetscBarrier(PETSC_NULL);
// stop log event
PetscLogEventEnd(metos3d->eventTimeStepPhi, 0, 0, 0, 0);
// debug
Metos3DDebug(metos3d, kDebugLevel, "Metos3DTimeStepPhi\n");
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DBGCStep(Metos3D *metos3d, PetscScalar t, PetscScalar dt, Vec *yin, Vec *yout, PetscInt nparam, PetscReal *u0)
{
// load
PetscInt nvecloc = metos3d->vectorLengthLocal;
PetscInt nprofloc = metos3d->profileCountLocal;
PetscInt *istartloc = metos3d->profileStartLocal;
PetscInt *iendloc = metos3d->profileEndLocal;
// bgc
PetscInt ntracer = metos3d->tracerCount;
PetscInt nbc = metos3d->boundaryConditionCount;
PetscInt ndc = metos3d->domainConditionCount;
// work vars
Vec *ybgcinBD = metos3d->ybgcinBD;
Vec *ybgcoutBD = metos3d->ybgcoutBD;
PetscInt iprof, nlayer;
PetscScalar *bcarray = PETSC_NULL;
PetscScalar *dcarray = PETSC_NULL;
PetscScalar *yinarray = PETSC_NULL;
PetscScalar *youtarray = PETSC_NULL;
PetscFunctionBegin;
// start log event
PetscLogEventBegin(metos3d->eventBGCStep, 0, 0, 0, 0);
// yin, copy to block diagonal, yout, zero entries
Metos3DUtilVecCopySeparateToDiagonal(metos3d, ntracer, nvecloc, yin, ybgcinBD);
VecZeroEntries(*ybgcoutBD);
// boundary condition step
// domain condition step
Metos3DBGCStepBoundaryCondition(metos3d, t);
Metos3DBGCStepDomainCondition(metos3d, t);
// bc, dc, yin, yout array
if (nbc > 0) VecGetArray(*metos3d->bgcbcBD, &bcarray);
if (ndc > 0) VecGetArray(*metos3d->bgcdcBD, &dcarray);
VecGetArray(*ybgcinBD, &yinarray);
VecGetArray(*ybgcoutBD, &youtarray);
// go through profiles
for (iprof = 0; iprof < nprofloc; iprof++)
{
// layer
nlayer = iendloc[iprof]-istartloc[iprof]+1;
//
// call BGC C API
//
#ifdef BGC
BGC(
(int*)&ntracer, // tracer count
(int*)&nlayer, // layer count
(int*)&nparam, // parameter count
(int*)&nbc, // boundary condition count
(int*)&ndc, // domain condition count
(double*)&dt, // time step
(double*)&youtarray[ntracer*(istartloc[iprof]-1)], // source minus sink
(double*)&t, // point in time
(double*)&yinarray [ntracer*(istartloc[iprof]-1)], // tracer
(double*)u0, // parameter
(double*)&bcarray [nbc*iprof], // boundary condition
(double*)&dcarray [ndc*(istartloc[iprof]-1)] // domain condition
);
#endif
}
// bc, dc, yin, yout array
if (nbc > 0) VecRestoreArray(*metos3d->bgcbcBD, &bcarray);
if (ndc > 0) VecRestoreArray(*metos3d->bgcdcBD, &dcarray);
VecRestoreArray(*ybgcinBD, &yinarray);
VecRestoreArray(*ybgcoutBD, &youtarray);
// copy back to separate vectors
Metos3DUtilVecCopyDiagonalToSeparate(metos3d, ntracer, nvecloc, ybgcoutBD, yout);
// stop log event
PetscLogEventEnd(metos3d->eventBGCStep, 0, 0, 0, 0);
// debug
Metos3DDebug(metos3d, kDebugLevel, "Metos3DBGCStep\n");
PetscFunctionReturn(0);
}
