/* PC_PROC function which applies the Sherman-Morrison-Woodbury formula */
static void
sherman(void *ctx, VEC *b0, VEC **x0)
{
SOLVER *solver = ctx;
VEC *U = ((void **)solver->mat->mv_data)[1];
SOLVER *solver1 = ((void **)solver->mat->mv_data)[2];
FLOAT *C = ((void **)solver->mat->mv_data)[3];
INT *pvt = ((void **)solver->mat->mv_data)[4];
VEC *x = *x0, *y;
INT i, j, n = U->map->nlocal, K = U->nvec;
FLOAT *z;
phgPrintf("Note: applying the Sherman-Morrison-Woodbury formula.\n");
/* x := inv(A) * b */
#if 0
memset(x->data, 0, n * sizeof(*x->data));
#endif
z = solver1->rhs->data; /* save solver1->rhs->data */
solver1->rhs->data = b0->data;
solver1->rhs->assembled = TRUE;
phgSolverVecSolve(solver1, FALSE, x);
solver1->rhs->data = z; /* restore solver1->rhs->data */
if (K == 0)
return;
/* z := U^t * x */
z = phgCalloc(K, sizeof(*z));
for (j = 0; j < K; j++)
for (i = 0; i < n; i++)
z[j] += U->data[j * n + i] * x->data[i];
#if USE_MPI
if (U->map->nprocs > 1) {
FLOAT *tmp = phgAlloc(K * sizeof(*tmp));
MPI_Allreduce(z, tmp, K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm);
phgFree(z);
z = tmp;
}
#endif /* USE_MPI */
/* z := C*z */
phgSolverDenseSV(K, C, pvt, 1, z);
/* y := inv(A) * (U*z) */
memset(solver1->rhs->data, 0, n * sizeof(*solver1->rhs->data));
for (i = 0; i < n; i++)
for (j = 0; j < K; j++)
solver1->rhs->data[i] += U->data[j * n + i] * z[j];
y = phgMapCreateVec(U->map, 1);
phgSolverVecSolve(solver1, FALSE, y);
phgFree(z);
/* x -= y */
for (i = 0; i < n; i++)
x->data[i] -= y->data[i];
phgVecDestroy(&y);
}
static void
do_test(MAP *rmap, MAP *cmap)
{
MAT *A;
VEC *x, *y;
INT i, j, m0, n0;
INT *cols = phgAlloc(cmap->nglobal * sizeof(*cols));
FLOAT *data = phgAlloc(cmap->nglobal * sizeof(*data));
A = phgMapCreateMat(rmap, cmap);
m0 = A->rmap->partition[A->rmap->rank];
n0 = A->cmap->partition[A->cmap->rank];
/* Matrix entries: A(I,J) = 1 + (I-1) + (J-1) */
for (i = m0; i < m0 + A->rmap->nlocal; i++) {
#if 0
/* Test MatAddEntry */
for (j = 0; j < A->cmap->nglobal; j++)
phgMatAddGlobalEntry(A, i, j, 1.0 + i + j);
#else
/* Test MatAddEntries */
for (j = 0; j < A->cmap->nglobal; j++) {
cols[j] = j;
data[j] = 1.0 + i + j;
}
phgMatAddGlobalEntries(A, 1, &i, A->cmap->nglobal, cols, data);
#endif
}
phgFree(cols);
phgFree(data);
phgMatAssemble(A);
phgInfo(-1, "y = A * x\n");
x = phgMapCreateVec(A->cmap, 1);
for (i = 0; i < x->map->nlocal; i++)
x->data[i] = 1.0 + i + n0;
phgVecAssemble(x);
y = phgMatVec(MAT_OP_N, 1.0, A, x, 0.0, NULL);
for (i = 0; i < y->map->nlocal; i++)
phgInfo(-1, " y->data[%d] = %lg\n", i + m0, (double)y->data[i]);
phgVecDestroy(&x);
phgVecDestroy(&y);
phgInfo(-1, "y = A' * x\n");
x = phgMapCreateVec(A->rmap, 1);
for (i = 0; i < x->map->nlocal; i++)
x->data[i] = 1.0 + i + m0;
phgVecAssemble(x);
y = phgMatVec(MAT_OP_T, 1.0, A, x, 0.0, NULL);
for (i = 0; i < y->map->nlocal; i++)
phgInfo(-1, " y->data[%d] = %lg\n", i + n0, (double)y->data[i]);
phgVecDestroy(&x);
phgVecDestroy(&y);
phgMatDestroy(&A);
}
/* callback function which computes A+UU^t */
static int
funcB(MAT_OP op, MAT *B, VEC *x, VEC *y)
/* computes y = op(A) * x */
{
MAT *A = ((void **)B->mv_data)[0];
VEC *U = ((void **)B->mv_data)[1];
INT i, j, k, n = U->map->nlocal, K = U->nvec;
/* compute y = op(A)*x */
phgMatVec(op, 1.0, A, x, 0.0, &y);
if (K == 0)
return 0;
if (op != MAT_OP_D) {
/* compute y += U*U^t*x */
FLOAT *z = phgCalloc(K, sizeof(*z));
/* z = U^t*x */
for (j = 0; j < K; j++)
for (i = 0; i < n; i++)
z[j] += U->data[j * n + i] * x->data[i];
#if USE_MPI
if (U->map->nprocs > 1) {
FLOAT *tmp = phgCalloc(K, sizeof(*tmp));
MPI_Allreduce(z, tmp, K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm);
phgFree(z);
z = tmp;
}
#endif /* USE_MPI */
/* y += U*z */
for (i = 0; i < n; i++)
for (j = 0; j < K; j++)
y->data[i] += U->data[j * n + i] * z[j];
phgFree(z);
return 0;
}
for (i = 0; i < n; i++) {
FLOAT d = 0.0;
for (j = 0; j < K; j++)
for (k = 0; k < K; k++)
d += U->data[j * n + i] * U->data[k * n + i];
y->data[i] += d * x->data[i];
}
return 0;
}
static void
setup_hypre_mat(MAT *A, HYPRE_IJMatrix *pa)
{
MAT_ROW *row;
INT i, j, d, I, n0, n, *cols;
assert(sizeof(INT) == sizeof(int) && sizeof(FLOAT)==sizeof(double));
phgMatUnpack(A);
HYPRE_IJMatrixCreate(A->rmap->comm,
A->rmap->partition[A->rmap->rank],
A->rmap->partition[A->rmap->rank + 1] - 1,
A->cmap->partition[A->cmap->rank],
A->cmap->partition[A->cmap->rank + 1] - 1,
pa);
HYPRE_IJMatrixSetObjectType(*pa, HYPRE_PARCSR);
HYPRE_IJMatrixInitialize(*pa);
I = A->rmap->partition[A->rmap->rank];
n0 = A->cmap->partition[A->cmap->rank];
n = 0;
cols = NULL;
for (i = 0, row = A->rows; i < A->rmap->nlocal; i++, row++, I++) {
if (row->ncols > 0) {
if (n < row->ncols) {
phgFree(cols);
cols = phgAlloc((n = row->ncols) * sizeof(*cols));
}
for (j = 0; j < row->ncols; j++) {
if ((d = row->cols[j]) < A->cmap->nlocal)
cols[j] = d + n0;
else
cols[j] = A->O2Gmap[d - A->cmap->nlocal];
}
HYPRE_IJMatrixSetValues(*pa, 1, &row->ncols, &I, cols, row->data);
}
}
phgFree(cols);
HYPRE_IJMatrixAssemble(*pa);
}
/*
* Jacobi smoother2:
* Implement using details of matvec
* ref: matvec in matvec.c
*
* */
void
mg_Jacobi2(MAT *A, VEC *x, VEC *b, int nsmooth, void *ctx)
{
INT i, j, k, n, *pc, *pc_offp;
FLOAT *pd, *pd_offp, *vx, *vx0, *vb;
FLOAT sum, omega = _p->smooth_damp;;
VEC *x0;
#if USE_MPI
FLOAT *offp_data = NULL;
#endif /* USE_MPI */
MagicCheck(VEC, x);
MagicCheck(VEC, b);
assert(x == NULL || x->nvec == 1);
assert(b == NULL || b->nvec == 1);
if (x != NULL && !x->assembled)
phgVecAssemble(x);
if (b != NULL && !b->assembled)
phgVecAssemble(b);
x0 = phgMapCreateVec(x->map, 1);
assert(A->type != PHG_DESTROYED);
if (!A->assembled)
phgMatAssemble(A);
assert(A->type != PHG_MATRIX_FREE);
phgMatPack(A);
#if USE_MPI
if (A->cmap->nprocs > 1) {
offp_data = phgAlloc(A->cinfo->rsize * sizeof(*offp_data));
}
#endif /* USE_MPI */
if (A->cmap->nlocal != A->rmap->nlocal ||
A->cmap->nlocal != x->map->nlocal ||
A->cmap->nlocal != b->map->nlocal)
phgError(1, "%s:%d: inconsistent matrix-vector.", __FILE__,
__LINE__);
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
/* iteration */
for (k = 0; k < nsmooth; k++) {
phgVecCopy(x, &x0);
/* multiply with local data */
vx = x->data;
vx0 = x0->data;
vb = b->data;
pc = A->packed_cols;
pd = A->packed_data;
if (A->cmap->nprocs > 1) {
pc_offp = A->packed_cols + A->rmap->nlocal + A->nnz_d;
pd_offp = A->packed_data + A->nnz_d;
} else {
pc_offp = NULL;
pd_offp = NULL;
}
for (i = 0; i < A->rmap->nlocal; i++) {
INT jcol;
FLOAT aa = 0., dx;
/* x_i = (b_i - \sum_{j ~= i} a_ij * x_j) / a_ii */
sum = vb[i];
/* local data */
if ((n = *(pc++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc++);
if (jcol != i) {
sum -= *(pd++) * vx0[jcol];
} else {
aa = *(pd++);
assert(fabs(aa) > 1e-14);
}
}
}
/* remote data */
if (pc_offp != NULL && (n = *(pc_offp++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc_offp++);
sum -= *(pd_offp++) * offp_data[jcol];
}
}
dx = sum / aa - vx[i];
vx[i] += omega * dx;
}
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
}
phgVecDestroy(&x0);
#if USE_MPI
phgFree(offp_data);
#endif /* USE_MPI */
return;
}
/*
* Gauss-Sidel smoother for vector:
*
* As GS:
* 1. exchange off proc data
* 2. smooth local dof
*
* Assumption:
* 1. unknow dof is vector of dim 3
* 2. Matrix block for vector is
* | a1 -b 0 |
* | b a2 0 |
* | 0 0 a3 |
*
* */
void
mg_GaussSidel_vec(MAT *A, VEC *x, VEC *b, int nsmooth, void *ctx)
{
INT i, j, k, l, n, *pc, *pc0, *pc_offp, nlocal;
FLOAT *pd, *pd0, *pd_offp, *vx, *vb;
size_t *ps;
FLOAT sum[Dim], omega = _p->smooth_damp;
#if USE_MPI
FLOAT *offp_data = NULL;
#endif /* USE_MPI */
MagicCheck(VEC, x);
MagicCheck(VEC, b);
assert(x == NULL || x->nvec == 1);
assert(b == NULL || b->nvec == 1);
if (x != NULL && !x->assembled)
phgVecAssemble(x);
if (b != NULL && !b->assembled)
phgVecAssemble(b);
assert(A->type != PHG_DESTROYED);
if (!A->assembled)
phgMatAssemble(A);
assert(A->type != PHG_MATRIX_FREE);
phgMatPack(A);
#if USE_MPI
if (A->cmap->nprocs > 1) {
offp_data = phgAlloc(A->cinfo->rsize * sizeof(*offp_data));
}
#endif /* USE_MPI */
if (A->cmap->nlocal != A->rmap->nlocal ||
A->cmap->nlocal != x->map->nlocal ||
A->cmap->nlocal != b->map->nlocal)
phgError(1, "%s:%d: inconsistent matrix-vector.", __FILE__,
__LINE__);
if (A->cmap->nlocal % Dim != 0)
phgError(1, "%s: assume vector dof of dim 3!\n", __FUNCTION__);
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
/* iteration */
for (l = 0; l < nsmooth; l++) {
INT i_start, i_add;
/* multiply with local data */
vx = x->data;
vb = b->data;
pc0 = A->packed_cols;
pd0 = A->packed_data;
ps = A->packed_ind;
nlocal = A->rmap->nlocal;
/*
* lexicographic order: low to high
* Note: low->high and high->low alternatively does not help.
* */
if (TRUE || l % 2 == 0) {
i_start = 0;
i_add = Dim;
} else {
i_start = nlocal - Dim;
i_add = -Dim;
}
//for (i = i_start; i < A->rmap->nlocal && i >= 0; i += i_add) {
for (i = 0; i < nlocal ; i += Dim) {
INT jcol;
FLOAT aa[Dim][Dim], det, dx[Dim];
memset(aa, 0, sizeof(aa));
sum[0] = vb[i ];
sum[1] = vb[i+1];
sum[2] = vb[i+2];
/* local data */
pc = pc0 + PACK_COL(ps, i);
pd = pd0 + PACK_DAT(ps, i);
for (k = 0; k < Dim; k++) {
if ((n = *(pc++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc++);
if (jcol < i || jcol > i+2) {
sum[k] -= *(pd++) * vx[jcol];
} else { /* offD, jcol = i,i+1,i+2 */
aa[k][jcol - i] = *(pd++);
}
}
}
}
/* remote data */
if (A->cmap->nprocs > 1) {
pc_offp = pc0 + PACK_COL_OFFP(ps, i, nlocal);
pd_offp = pd0 + PACK_DAT_OFFP(ps, i, nlocal);
for (k = 0; k < Dim; k++) {
if ((n = *(pc_offp++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc_offp++);
sum[k] -= *(pd_offp++) * offp_data[jcol];
}
}
}
}
/* solve */
det = (aa[0][0] * aa[1][1] - aa[0][1] * aa[1][0]);
assert(Fabs(det) > 1e-12);
det = 1. / det;
dx[0] = (aa[1][1] * sum[0] - aa[0][1] * sum[1]) * det - vx[i ];
dx[1] = (aa[0][0] * sum[1] - aa[1][0] * sum[1]) * det - vx[i+1];
dx[2] = (1./aa[2][2] * sum[2]) - vx[i+2];
vx[i ] += omega * dx[0];
vx[i+1] += omega * dx[1];
vx[i+2] += omega * dx[2];
}
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
}
#if USE_MPI
phgFree(offp_data);
#endif /* USE_MPI */
return;
}
/*
* Gauss-Sidel smoother2:
*
* 1. interior dof (!REMOTE) is smoothed, and then offp data is updated;
* 2. proc boundary dof (REMOTE) is smoothed, and then offp data is updated.
*
* Note: need types_vec.
* */
void
mg_GaussSidel2(MAT *A, VEC *x, VEC *b, int nsmooth, void *ctx)
{
MG_LEVEL *ml = (MG_LEVEL *)ctx;
INT i, j, k, n, *pc, *pc_offp;
FLOAT *pd, *pd_offp, *vx, *vb;
FLOAT sum, omega = _p->smooth_damp;;
#if USE_MPI
FLOAT *offp_data = NULL;
#endif /* USE_MPI */
MagicCheck(VEC, x);
MagicCheck(VEC, b);
assert(x == NULL || x->nvec == 1);
assert(b == NULL || b->nvec == 1);
if (x != NULL && !x->assembled)
phgVecAssemble(x);
if (b != NULL && !b->assembled)
phgVecAssemble(b);
assert(A->type != PHG_DESTROYED);
if (!A->assembled)
phgMatAssemble(A);
assert(A->type != PHG_MATRIX_FREE);
phgMatPack(A);
#if USE_MPI
if (A->cmap->nprocs > 1) {
offp_data = phgAlloc(A->cinfo->rsize * sizeof(*offp_data));
}
#endif /* USE_MPI */
if (A->cmap->nlocal != A->rmap->nlocal ||
A->cmap->nlocal != x->map->nlocal ||
A->cmap->nlocal != b->map->nlocal)
phgError(1, "%s:%d: inconsistent matrix-vector.", __FILE__,
__LINE__);
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
/* iteration */
for (k = 0; k < nsmooth; k++) {
/* multiply with local data */
vx = x->data;
vb = b->data;
/* First, interior dof (!REMOTE) is smoothed, and then offp data is updated */
pc = A->packed_cols;
pd = A->packed_data;
if (A->cmap->nprocs > 1) {
pc_offp = A->packed_cols + A->rmap->nlocal + A->nnz_d;
pd_offp = A->packed_data + A->nnz_d;
} else {
pc_offp = NULL;
pd_offp = NULL;
}
for (i = 0; i < A->rmap->nlocal; i++) {
INT jcol;
FLOAT aa = 0., dx;
if (ml->types_vec[i] & REMOTE) {
if ((n = *(pc++)) != 0) {
pc += n;
pd += n;
}
if (pc_offp != NULL && (n = *(pc_offp++)) != 0) {
pc_offp += n;
pd_offp += n;
}
continue;
}
sum = vb[i];
/* local data */
if ((n = *(pc++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc++);
if (jcol != i) {
sum -= *(pd++) * vx[jcol];
} else {
aa = *(pd++);
assert(fabs(aa) > 1e-14);
}
}
}
/* remote data */
if (pc_offp != NULL && (n = *(pc_offp++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc_offp++);
sum -= *(pd_offp++) * offp_data[jcol];
}
}
dx = sum / aa - vx[i];
vx[i] += omega * dx;
}
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
/* Second, proc boundary dof (!REMOTE) is smoothed, and then offp data is updated */
pc = A->packed_cols;
pd = A->packed_data;
if (A->cmap->nprocs > 1) {
pc_offp = A->packed_cols + A->rmap->nlocal + A->nnz_d;
pd_offp = A->packed_data + A->nnz_d;
} else {
pc_offp = NULL;
pd_offp = NULL;
}
for (i = 0; i < A->rmap->nlocal; i++) {
INT jcol;
FLOAT aa = 0., dx;
if (!(ml->types_vec[i] & REMOTE)) {
if ((n = *(pc++)) != 0) {
pc += n;
pd += n;
}
if (pc_offp != NULL && (n = *(pc_offp++)) != 0) {
pc_offp += n;
pd_offp += n;
}
continue;
}
sum = vb[i];
/* local data */
if ((n = *(pc++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc++);
if (jcol != i) {
sum -= *(pd++) * vx[jcol];
} else {
aa = *(pd++);
assert(fabs(aa) > 1e-14);
}
}
}
/* remote data */
if (pc_offp != NULL && (n = *(pc_offp++)) != 0) {
for (j = 0; j < n; j++) {
jcol = *(pc_offp++);
sum -= *(pd_offp++) * offp_data[jcol];
}
}
dx = sum / aa - vx[i];
vx[i] += omega * dx;
}
#if USE_MPI
if (A->cmap->nprocs > 1) {
phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data);
phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data);
}
#endif /* USE_MPI */
}
#if USE_MPI
phgFree(offp_data);
#endif /* USE_MPI */
return;
}
/****************************************************************
* Build RHS which is the residual of the nonlinear system.
***************************************************************/
void
phgNSBuildSolverURHS(NSSolver *ns)
{
GRID *g = ns->g;
SIMPLEX *e;
SOLVER *solver_u = ns->solver_u;
int i, k, l, q, s;
FLOAT *dt = ns->dt;
BOOLEAN tstep_minus = (ns->u[-1] != NULL);
VEC *vec_rhs = phgMapCreateVec(solver_u->rhs->map, 1);
FLOAT Theta = _nsp->Theta, nu = _nsp->nu, Thet1;
int viscosity_type = ns->viscosity_type;
SURF_BAS *surf_bas = ns->surf_bas;
DOF *surf_dof = surf_bas->dof;
BOOLEAN *rotated = surf_bas->rotated;
const FLOAT *Trans = DofData(surf_dof);
#if STEADY_STATE
assert(fabs(Theta - 1) < 1e-12);
Thet1 = 0; Unused(Thet1);
Unused(dt);
#else
Thet1 = 1 - Theta;
Unused(dt);
#endif /* STEADY_STATE */
phgPrintf(" DB_mask: [");
for (k = 0; k < Dim; k++)
phgPrintf("%d ", ns->u[1]->DB_masks[k]);
phgPrintf("] ");
nu_max = -1e10;
nu_min = +1e10;
phgVecDisassemble(vec_rhs);
ForAllElements(g, e) {
int M = ns->u[1]->type->nbas; /* num of bases of Velocity */
int N = ns->p[1]->type->nbas; /* num of bases of Pressure */
int order = DofTypeOrder(ns->u[1], e) * 3 - 1; /* Note:
* quad order is really high here,
* highest order term (u \nabla u, phi) */
FLOAT bufu[M], bufp[N], rhsu[M][Dim], rhsp[N];
INT Iu[M][Dim], Ip[N];
QUAD *quad;
FLOAT vol, area, det;
const FLOAT *w, *p, *normal,
**vu, *vu_queue[3],
*vf[2], *gu[2], *vp[2], *vw, *vT;
FLOAT *vf_cache[2];
vu = vu_queue + 1;
quad = phgQuadGetQuad3D(order);
vu[0] = phgQuadGetDofValues(e, ns->u[0], quad); /* u^{n} */
vp[0] = phgQuadGetDofValues(e, ns->p[0], quad); /* p^{n} */
gu[0] = phgQuadGetDofValues(e, ns->gradu[0], quad); /* grad u^{n} */
vw = phgQuadGetDofValues(e, ns->wind, quad); /* wind */
vT = phgQuadGetDofValues(e, ns->T[1], quad); /* T^{n} */
if (tstep_minus) {
vu[-1] = phgQuadGetDofValues(e, ns->u[-1], quad); /* u^{n-1} */
} else {
vu[-1] = vu[0];
}
#if STEADY_STATE || TIME_DEP_NON
vu[1] = phgQuadGetDofValues(e, ns->u[1], quad); /* u^{n+1} */
gu[1] = phgQuadGetDofValues(e, ns->gradu[1], quad); /* grad u^{n} */
vp[1] = phgQuadGetDofValues(e, ns->p[1], quad); /* p^{n+1} */
#else
TIME_DEP_LINEAR_ENTRY; /* Unavailable */
#endif /* STEADY_STATE || TIME_DEP_NON */
Unused(l);
Unused(vf); Unused(vf_cache);
if (!_nsp->no_extern_source) {
/* cache f values */
for (l = 0; l < 2; l++) {
const FLOAT *cache;
size_t cache_size;
setFuncTime(ns->time[l]); /* set static time in ins-test.c */
/* cache f */
cache_size = Dim * quad->npoints * sizeof(FLOAT);
cache = phgQuadGetFuncValues(g, e, Dim, func_f, quad);
vf[l] = vf_cache[l] = phgAlloc(cache_size);
memcpy(vf_cache[l], cache, cache_size);
phgQuadGetFuncValues(NULL, NULL, 0, NULL, NULL); /* clear cache */
}
}
/* Global Matrix */
Bzero(rhsu); Bzero(rhsp);
p = quad->points;
w = quad->weights;
for (q = 0; q < quad->npoints; q++) {
phgGeomGetCurvedJacobianAtLambda(g, e, p, &det);
vol = fabs(det / 6.);
/* rhs u */
for (i = 0; i < M; i++) {
/* interior node or Neumann */
const FLOAT *gi_u = phgQuadGetBasisValues(e, ns->u[1], i, quad) + q; /* phi_i */
const FLOAT *ggi_u = phgQuadGetBasisCurvedGradient(e, ns->u[1], i, quad, q); /* grad phi_i */
for (k = 0; k < Dim; k++) {
#if ICE_BENCH_TEST
nu = get_effective_viscosity(gu[1], 0, 0, viscosity_type);
FLOAT eu[DDim];
MAT3_SYM(gu[1], eu);
rhsu[i][k] += vol*(*w) * EQU_SCALING * (- nu * INNER_PRODUCT(eu+k*Dim, ggi_u)
+ (*vp[1]) * *(ggi_u+k) * LEN_SCALING * PRES_SCALING
); /* left */
if (k == Z_DIR) {
const FLOAT rho = RHO_ICE;
const FLOAT grav = GRAVITY;
const FLOAT a = SEC_PER_YEAR;
const FLOAT f = rho*grav * EQU_SCALING * LEN_SCALING2;
Unused(a);
rhsu[i][k] += vol*(*w) * (-f * (*gi_u)
); /* right */
}
#elif ESIMINT_TEST || \
HEINO_TEST || \
TEST_CASE == ICE_GREEN_LAND
nu = get_effective_viscosity(gu[1], *vT, 0, viscosity_type);
FLOAT eu[DDim];
MAT3_SYM(gu[1], eu);
rhsu[i][k] += vol*(*w) * EQU_SCALING * (- nu * INNER_PRODUCT(eu+k*Dim, ggi_u)
+ (*vp[1]) * *(ggi_u+k) * LEN_SCALING * PRES_SCALING
); /* left */
if (k == Z_DIR) {
const FLOAT rho = RHO_ICE;
const FLOAT grav = GRAVITY;
const FLOAT a = SEC_PER_YEAR;
const FLOAT f = rho*grav * EQU_SCALING * LEN_SCALING2;
Unused(a);
rhsu[i][k] += vol*(*w) * (-f * (*gi_u)
); /* right */
}
#elif STEADY_STATE
rhsu[i][k] += vol*(*w) * (- nu * INNER_PRODUCT(gu[1]+k*Dim, ggi_u)
+ (*vp[1]) * *(ggi_u+k)
); /* left */
if (!_nsp->no_extern_source)
rhsu[i][k] += vol*(*w) * (*(vf[1]+k) * (*gi_u)
); /* right */
#elif TIME_DEP_NON
rhsu[i][k] -= vol*(*w) * ((vu[1][k] - vu[0][k]) * (*gi_u) / dt[0]
+ Theta * (nu * INNER_PRODUCT(gu[1]+k*Dim, ggi_u)
)
- (*vp[1]) * *(ggi_u+k)
+ Thet1 * (nu * INNER_PRODUCT(gu[0]+k*Dim, ggi_u)
)
); /* left */
if (!_nsp->no_extern_source)
rhsu[i][k] += vol*(*w) * (Theta * *(vf[1]+k) * (*gi_u)
+ Thet1 * *(vf[0]+k) * (*gi_u)
); /* right */
#else
TIME_DEP_LINEAR_ENTRY; /* Unavailable */
#endif /* STEADY_STATE */
}
}
/* rhs p */
for (i = 0; i < N; i++) {
const FLOAT *gi_p = phgQuadGetBasisValues(e, ns->p[1], i, quad) + q; /* psi_i */
FLOAT divu1 = gu[1][0] + gu[1][4] + gu[1][8];
//FLOAT divu0 = gu[0][0] + gu[0][4] + gu[0][8];
rhsp[i] += vol*(*w) * (divu1 * (*gi_p)
);
}
if (tstep_minus)
vu[-1] += Dim;
#if STEADY_STATE || TIME_DEP_NON
vu[1] += Dim;
gu[1] += Dim*Dim;
vp[1]++;
#else
TIME_DEP_LINEAR; /* Unavailable */
#endif /* STEADY_STATE || TIME_DEP_NON */
vu[0] += Dim;
gu[0] += Dim * Dim;
vp[0]++;
vw += Dim;
if (!_nsp->no_extern_source) {
vf[0] += Dim; vf[1] += Dim;
}
vT++;
w++; p += Dim + 1;
}
if (!_nsp->no_extern_source) {
phgFree(vf_cache[0]);
phgFree(vf_cache[1]);
}
normal = NULL; Unused(normal);
area = 0; Unused(area);
if (!_nsp->enclosed_flow) {
/* slip boundary */
for (s = 0; s < NFace; s++) {
if (e->bound_type[s] & INFLOW) {
int v0, v1, v2;
int nbas_face = NbasFace(ns->u[1]);
SHORT bases[nbas_face];
FLOAT lambda[Dim + 1], x,y,z, beta;
order = DofTypeOrder(ns->u[1], e) * 3 - 1;
phgDofGetBasesOnFace(ns->u[1], e, s, bases);
v0 = GetFaceVertex(s, 0);
v1 = GetFaceVertex(s, 1);
v2 = GetFaceVertex(s, 2);
lambda[s] = 0.;
area = phgGeomGetFaceArea(g, e, s);
normal = phgGeomGetFaceOutNormal(g, e, s);
quad = phgQuadGetQuad2D(order);
p = quad->points;
w = quad->weights;
for (q = 0; q < quad->npoints; q++) {
FLOAT vu[Dim];
lambda[v0] = *(p++);
lambda[v1] = *(p++);
lambda[v2] = *(p++);
phgGeomLambda2XYZ(g, e, lambda, &x, &y, &z);
func_beta(x, y, z, &beta);
phgDofEval(ns->u[1], e, lambda, vu);
for (i = 0; i < nbas_face; i++) {
int ii = bases[i];
FLOAT gi_u =
*ns->u[1]->type->BasFuncs(ns->u[1], e, ii, ii + 1, lambda);
for (k = 0; k < Dim; k++) {
#if STEADY_STATE
rhus[ii][k] += 0.;
#elif TIME_DEP_NON
# if USE_SLIDING_BC
abort();
rhsu[ii][k] += SIGN_FRICTION * area*(*w) * beta * vu[k] * (gi_u)
* EQU_SCALING * LEN_SCALING;
# else
Unused(gi_u);
# endif
#else
TIME_DEP_LINEAR_ENTRY; /* Unavailable */
#endif /* STEADY_STATE */
}
} /* end of bas_i */
w++;
} /* end of quad point */
} /* end of face outflow */
} /* end of all outflow face in element */
} /* end out flow boundary */
#if USE_SLIDING_BC
/* Rotate bases */
for (i = 0; i < M; i++) {
INT id = phgDofMapE2D(surf_dof, e, i * (Dim*Dim)) / (Dim*Dim);
if (!rotated[id])
continue;
const FLOAT *trans = Trans + id*(Dim*Dim);
trans_left(&rhsu[i][0], 1, 1, trans);
}
#else
Unused(Trans);
Unused(rotated);
#endif
/* Map: Element -> system */
for (i = 0; i < M; i++)
for (k = 0; k < Dim; k++)
Iu[i][k] = phgMapE2L(solver_u->rhs->map, 0, e, i * Dim + k);
for (i = 0; i < N; i++)
Ip[i] = phgMapE2L(solver_u->rhs->map, 1, e, i);
/* set velocity dirichlet bdry */
FLOAT tmp[Dim];
for (i = 0; i < M; i++)
for (k = 0; k < Dim; k++)
if (phgDofDirichletBC_(ns->u[1], e, i*Dim+k, NULL, bufu, tmp,
DOF_PROJ_NONE)) {
rhsu[i][k] = 0.;
}
#if STEADY_STATE || TIME_DEP_NON
/* set pressure dirichlet bdry for pinned point */
for (i = 0; i < N; i++)
if (phgDofDirichletBC(ns->p[1], e, i, NULL, bufp, &rhsp[i],
DOF_PROJ_NONE)) {
if (!_nsp->enclosed_flow)
phgError(1, "No dirichlet bc for Unenclosed flow!\n");
if (_nsp->pin_node) {
# if PIN_AT_ROOT
if (g->rank != 0)
phgError(1, "Pinned node only on rank 0!\n");
if (g, e->verts[i] != ns->pinned_node_id)
phgError(1, "Build rhs: pinned node e:%d, bas:%d, [%d] and [%d] "
"doesn't coincide when build RHS!\n",
e->index, i, e->verts[i], ns->pinned_node_id);
# else
if (GlobalVertex(g, e->verts[i]) != ns->pinned_node_id)
phgError(1, "Build rhs: pinned node e:%d, bas:%d, [%d] and [%d] "
"doesn't coincide when build RHS!\n",
e->index, i, e->verts[i], ns->pinned_node_id);
# endif /* PIN_AT_ROOT */
}
}
#else
TIME_DEP_LINEAR; /* Unavailable */
#endif /* STEADY_STATE || TIME_DEP_NON */
/* Global res */
phgVecAddEntries(vec_rhs, 0, M * Dim, Iu[0], &rhsu[0][0]);
phgVecAddEntries(vec_rhs, 0, N, Ip, rhsp);
} /* end element */
DOF *
phgDofCopy_(DOF *src, DOF **dest_ptr, DOF_TYPE *newtype,
const char *name, const char *srcfile, int srcline)
/* returns a new DOF whose type is 'newtype', and whose values are evaluated
* using 'src' (a copy of src). */
{
GRID *g = src->g;
SIMPLEX *e;
FLOAT w = 1.0, *basvalues = NULL, *a, *d, *buffer = NULL;
int i, k, dim, dim_new, count = 0, nvalues, nd;
INT n;
DOF *wgts = NULL;
DOF *dest = (dest_ptr == NULL ? NULL : *dest_ptr);
BYTE *flags0, *flags;
char *auto_name;
DOF_TYPE *oldtype;
BOOLEAN basflag = FALSE;
MagicCheck(DOF, dest)
if (dest != NULL && newtype != NULL && dest->type != newtype) {
phgDofFree(&dest);
dest = NULL;
}
dim = DofDim(src);
/* the name of dest */
if ((auto_name = (void *)name) == NULL) {
#if 0
auto_name = phgAlloc(strlen(src->name) + 8 + 1);
sprintf(auto_name, "copy of %s", src->name);
#else
auto_name = strdup(src->name);
#endif
}
if (dest == NULL) {
if (newtype == NULL)
newtype = src->type;
dim_new = (newtype == NULL ? DofTypeDim(src) : newtype->dim);
assert(dim % dim_new == 0);
dest = phgDofNew_(g, newtype, newtype == NULL ? src->hp : NULL,
dim / dim_new, auto_name, DofNoAction,
srcfile, srcline);
if (dest_ptr != NULL)
*dest_ptr = dest;
}
else {
assert(dim == DofDim(dest));
phgFree(dest->name); dest->name = strdup(auto_name);
dest->srcfile = srcfile;
dest->srcline = srcline;
phgFree(dest->cache_func); dest->cache_func = NULL;
newtype = dest->type;
if (!SpecialDofType(newtype))
memset(dest->data, 0, DofGetDataCount(dest) * sizeof(*dest->data));
}
if (auto_name != name)
phgFree(auto_name);
phgDofClearCache(NULL, dest, NULL, NULL, FALSE);
dest->DB_mask = src->DB_mask;
if (src->DB_masks != NULL) {
if (dest->DB_masks == NULL)
dest->DB_masks = phgAlloc(dest->dim * sizeof(*dest->DB_masks));
memcpy(dest->DB_masks, src->DB_masks, dest->dim * sizeof(*dest->DB_masks));
}
dest->invariant = src->invariant;
if (SpecialDofType(newtype)) {
assert(newtype == src->type);
dest->userfunc = src->userfunc;
dest->userfunc_lambda = src->userfunc_lambda;
if (newtype == DOF_CONSTANT)
memcpy(dest->data, src->data, dim * sizeof(*dest->data));
return dest;
}
if ((newtype != NULL && newtype == src->type) ||
(newtype == NULL && src->hp == dest->hp)) {
/* simply duplicate the data */
size_t size = DofGetDataCount(dest);
if (size > 0)
memcpy(dest->data, src->data, sizeof(FLOAT) * size);
dest->userfunc = src->userfunc;
dest->userfunc_lambda = src->userfunc_lambda;
return dest;
}
if (src->type == DOF_ANALYTIC) {
if (src->userfunc_lambda != NULL)
phgDofSetDataByLambdaFunction(dest, src->userfunc_lambda);
else
phgDofSetDataByFunction(dest, src->userfunc);
return dest;
}
if (newtype != NULL && newtype->BasFuncs == NULL)
phgError(1, "phgDofCopy: basis funcs for DOF type \"%s\" undefined.\n",
newtype->name);
dest->userfunc = src->userfunc;
dest->userfunc_lambda = src->userfunc_lambda;
oldtype = src->type;
if (oldtype == NULL)
oldtype = src->hp->info->types[src->hp->info->min_order];
if (!SpecialDofType(oldtype) && newtype != NULL && newtype->points != NULL
&& !DofIsHP(src)) {
basflag = TRUE;
count = oldtype->nbas * oldtype->dim;
basvalues = phgAlloc(newtype->nbas * count * sizeof(*basvalues));
if (oldtype->invariant == TRUE)
get_bas_funcs(src, dest, src->g->roots, basvalues);
}
if (newtype == NULL)
newtype = dest->hp->info->types[dest->hp->max_order];
flags0 = phgCalloc((newtype->np_vert > 0 ? g->nvert : 0)
+ (newtype->np_edge > 0 ? g->nedge : 0)
+ (newtype->np_face > 0 ? g->nface : 0),
sizeof(*flags0));
if (!SpecialDofType(oldtype) && oldtype->continuity < 0) {
static DOF_TYPE DOF_WGTS = {DofCache,
"Weights", NULL, NULL, NULL, NULL, NULL, NULL, NULL,
phgDofInitFuncPoint, NULL, NULL, NULL, FE_None,
FALSE, FALSE, -1, 0, 0, -1, 1, 0, 0, 0, 0
};
DOF_WGTS.np_vert = (newtype->np_vert > 0) ? 1 : 0;
DOF_WGTS.np_edge = (newtype->np_edge > 0) ? 1 : 0;
DOF_WGTS.np_face = (newtype->np_face > 0) ? 1 : 0;
DOF_WGTS.nbas = DOF_WGTS.np_vert * NVert + DOF_WGTS.np_edge * NEdge +
DOF_WGTS.np_face * NFace;
if (DOF_WGTS.nbas > 0) {
/* Other cases will be implemented later when really needed */
wgts = phgDofNew(g, &DOF_WGTS, 1, "weights", DofNoAction);
phgDofSetDataByValue(wgts, 0.0);
phgDofSetDataByValue(dest, 0.0);
/* allocate buffer for storing weighted vertex/edge/face data */
if ((n = DofGetDataCount(dest) - DofGetElementDataCount(dest)) > 0)
buffer = phgCalloc(n, sizeof(*buffer));
}
}
nvalues = dest->dim;
cache_dof = src;
bas_count = count;
ForAllElements(g, e) {
if (wgts != NULL) {
#if 0
/* use element volume as weight */
w = phgGeomGetVolume(g, e);
#else
/* use 1 as weight */
w = 1.0;
#endif
}
if (basflag && oldtype->invariant == FALSE)
get_bas_funcs(src, dest, e, basvalues);
bas = basvalues;
flags = flags0;
if (DofIsHP(dest))
newtype = dest->hp->info->types[dest->hp->elem_order[e->index]];
if (newtype->np_vert > 0) {
nd = nvalues * newtype->np_vert;
for (k = 0; k < NVert; k++) {
if (flags[n = e->verts[k]] && wgts == NULL) {
/* note: count==0 and bas==NULL for variable order DOF */
bas += count * newtype->np_vert;
continue;
}
flags[n] = TRUE;
a = DofVertexData(dest, n);
newtype->InitFunc(dest, e, VERTEX, k, NULL, func, NULL, a,
NULL);
if (wgts != NULL) {
d = buffer + (a - DofData(dest));
for (i = 0; i < nd; i++)
*(d++) += *(a++) * w;
*DofVertexData(wgts, n) += w;
}
}
flags += g->nvert;
}
if (newtype->np_edge > 0) {
nd = nvalues * newtype->np_edge;
for (k = 0; k < NEdge; k++) {
if (flags[n = e->edges[k]] && wgts == NULL) {
/* note: count==0 and bas==NULL for variable order DOF */
bas += count * newtype->np_edge;
continue;
}
flags[n] = TRUE;
a = DofEdgeData(dest, n);
newtype->InitFunc(dest, e, EDGE, k, NULL, func, NULL, a,
NULL);
if (wgts != NULL) {
d = buffer + (a - DofData(dest));
if (DofIsHP(dest))
nd = dest->dim * (dest->hp->edge_index[n + 1] -
dest->hp->edge_index[n]);
for (i = 0; i < nd; i++)
*(d++) += *(a++) * w;
*DofEdgeData(wgts, n) += w;
}
}
flags += g->nedge;
}
if (newtype->np_face > 0) {
nd = nvalues * newtype->np_face;
for (k = 0; k < NFace; k++) {
if (flags[n = e->faces[k]] && wgts == NULL) {
/* note: count==0 and bas==NULL for variable order DOF */
bas += count * newtype->np_face;
continue;
}
flags[n] = TRUE;
a = DofFaceData(dest, n);
newtype->InitFunc(dest, e, FACE, k, NULL, func, NULL, a,
NULL);
if (wgts != NULL) {
d = buffer + (a - DofData(dest));
if (DofIsHP(dest))
nd = dest->dim * (dest->hp->face_index[n + 1] -
dest->hp->face_index[n]);
for (i = 0; i < nd; i++)
*(d++) += *(a++) * w;
*DofFaceData(wgts, n) += w;
}
}
}
if (newtype->np_elem > 0) {
a = DofElementData(dest, e->index);
newtype->InitFunc(dest, e, ELEMENT, 0, NULL, func, NULL, a,
NULL);
}
}
phgFree(basvalues);
phgFree(flags0);
if (wgts == NULL)
return dest;
if ((n = DofGetDataCount(dest) - DofGetElementDataCount(dest)) > 0) {
memcpy(DofData(dest), buffer, n * sizeof(*buffer));
phgFree(buffer);
}
if (DofIsHP(dest))
newtype = dest->hp->info->types[dest->hp->max_order];
a = DofData(dest);
d = DofData(wgts);
#if USE_MPI
/* FIXME: directly interacting with the vector assembly code in solver.c
is more efficient */
if (g->nprocs > 1) {
SOLVER *solver = phgSolverCreate(SOLVER_BUILTIN, dest, NULL);
INT K = 0, I;
int j;
if (newtype->np_vert > 0) {
nd = dest->count_vert;
for (n = 0; n < g->nvert; n++, d++) {
if (g->types_vert[n] == UNREFERENCED) {
K += nd;
a += nd;
continue;
}
for (j = 0; j < nd; j++, K++, a++) {
I = phgSolverMapD2L(solver, 0, K);
assert(I >= 0 && I < solver->mat->rmap->localsize);
phgSolverAddRHSEntry(solver, I, *a);
phgSolverAddMatrixEntry(solver, I, I, *d);
}
}
}
if (newtype->np_edge > 0) {
nd = nvalues * newtype->np_edge;
for (n = 0; n < g->nedge; n++, d++) {
if (DofIsHP(dest))
nd = dest->dim * (dest->hp->edge_index[n + 1] -
dest->hp->edge_index[n]);
if (g->types_edge[n] == UNREFERENCED) {
K += nd;
a += nd;
continue;
}
for (j = 0; j < nd; j++, K++, a++) {
I = phgSolverMapD2L(solver, 0, K);
assert(I >= 0 && I < solver->mat->rmap->localsize);
phgSolverAddRHSEntry(solver, I, *a);
phgSolverAddMatrixEntry(solver, I, I, *d);
}
}
}
if (newtype->np_face > 0) {
nd = nvalues * newtype->np_face;
for (n = 0; n < g->nface; n++, d++) {
if (DofIsHP(dest))
nd = dest->dim * (dest->hp->face_index[n + 1] -
dest->hp->face_index[n]);
if (g->types_face[n] == UNREFERENCED) {
K += nd;
a += nd;
continue;
}
for (j = 0; j < nd; j++, K++, a++) {
I = phgSolverMapD2L(solver, 0, K);
assert(I >= 0 && I < solver->mat->rmap->localsize);
phgSolverAddRHSEntry(solver, I, *a);
phgSolverAddMatrixEntry(solver, I, I, *d);
}
}
}
if (newtype->np_elem > 0) {
nd = nvalues * newtype->np_elem;
for (n = 0; n < g->nelem; n++) {
if (DofIsHP(dest))
nd = dest->dim * (dest->hp->elem_index[n + 1] -
dest->hp->elem_index[n]);
if (g->types_elem[n] == UNREFERENCED) {
K += nd;
a += nd;
continue;
}
for (j = 0; j < nd; j++, K++, a++) {
I = phgSolverMapD2L(solver, 0, K);
assert(I >= 0 && I < solver->mat->rmap->localsize);
phgSolverAddRHSEntry(solver, I, *a);
phgSolverAddMatrixEntry(solver, I, I, 1.0);
}
}
}
phgSolverSolve(solver, TRUE, dest, NULL);
phgSolverDestroy(&solver);
phgDofFree(&wgts);
return dest;
}
#endif
if (newtype->np_vert > 0) {
k = nvalues * newtype->np_vert;
for (n = 0; n < g->nvert; n++) {
if ((w = *(d++)) == 0.0) {
a += k;
}
else if (k == 1) {
*(a++) /= w;
}
else {
w = 1.0 / w;
for (i = 0; i < k; i++)
*(a++) *= w;
}
}
}
if (newtype->np_edge > 0) {
k = nvalues * newtype->np_edge;
for (n = 0; n < g->nedge; n++) {
if (DofIsHP(dest))
k = dest->dim * (dest->hp->edge_index[n + 1] -
dest->hp->edge_index[n]);
if ((w = *(d++)) == 0.0) {
a += k;
}
else if (k == 1) {
*(a++) /= w;
}
else {
w = 1.0 / w;
for (i = 0; i < k; i++)
*(a++) *= w;
}
}
}
if (newtype->np_face > 0) {
k = nvalues * newtype->np_face;
for (n = 0; n < g->nface; n++) {
if (DofIsHP(dest))
k = dest->dim * (dest->hp->face_index[n + 1] -
dest->hp->face_index[n]);
if ((w = *(d++)) == 0.0) {
a += k;
}
else if (k == 1) {
*(a++) /= w;
}
else {
w = 1.0 / w;
for (i = 0; i < k; i++)
*(a++) *= w;
}
}
}
phgDofFree(&wgts);
return dest;
}
int
main(int argc, char *argv[])
{
GRID *g;
SIMPLEX *e;
DOF **u, **p, **T, **gradu,
*eu, *ep, *egradu, *ediv, *eT,
*dH = NULL;
FLOAT Time, *dt, res, non_du, non_dp, non_dT;
INT tstep = 0, nelem;
char mesh_file[100], hostname[256],
data_file[100], data_u[100], data_p[100], data_T[100], data_Crd[100];
size_t mem, mem_peak;
int verb;
double tt[3], tt1[3];
/* ---------- NS ---------- */
NSSolver *ns = NULL;
SURF_BAS *surf_bas = NULL;
LAYERED_MESH *gL = NULL;
//GEO_INFO *geo = NULL;
/* MG_BLOCK_DOFS *bk = NULL; */
/* ================================================================================
*
* Initialize Grid & parameters
*
* ================================================================================
*/
/* Global (static) options */
Unused(verb);
ns_params = phgParametersCreate();
phgInit(&argc, &argv);
phgOptionsShowUsed();
g = phgNewGrid(-1);
//phgSetPeriodicity(g, ns_params->periodicity);
phgImportSetBdryMapFunc(my_bc_map);
if (ns_params->resume) {
phgResumeStage(g, &Time, &tstep, mesh_file, data_file);
phgPrintf("================================\n\n");
phgPrintf("* RESUME from time:%E, tstep:%d\n", Time, tstep);
phgPrintf("* mesh:%s\n", mesh_file);
phgPrintf("* data:%s\n", data_file);
phgPrintf("================================\n");
if (!phgImport(g, mesh_file, FALSE))
phgError(1, "can't read file \"%s\".\n", ns_params->fn);
}
else {
phgPrintf("Using mesh: %s\n", ns_params->fn);
if (!phgImport(g, ns_params->fn, FALSE))
phgError(1, "can't read file \"%s\".\n", ns_params->fn);
}
checkBdry(g);
elapsed_time(g, FALSE, 0.); /* reset timer */
gethostname(hostname, sizeof(hostname));
printf("#%5d# runing PID %5d on %s \n", phgRank, getpid(), hostname);
NsSolver_Options();
phgPrintf(" Pre-refine & repartition ");
phgRefineAllElements(g, ns_params->pre_refines);
/* Set Reynolds number */
Time = ns_params->time_start; /* default: 0 */
setFuncTime(Time);
setFlowParameter(ns_params->Re, ns_params->nu, Time);
/* ================================================================================
*
* build ice grid
*
* ================================================================================
*/
iceInit(g, &gL);
phgPrintf("Geometry initialization done!\n");
phgExportVTK(g, "ice_domain.vtk", NULL, NULL);
/* ================================================================================
*
* Create INS solver
*
* ================================================================================
*/
/* Note: pointers u, p, gradu, dt
* DIRECTLY access private member of INS solver.
*/
phgPrintf(" Create INS solver");
tstep = 1; /* time step start at 1 */
setFuncTime(Time); /* in file ins-bc.c: static */
//data_sur_T = read_txt_data("./LAS_temp_for_mesh.txt");
ns = phgNSCreate(g, ns_params);
//get_mask_bot(ns);
ns->time[1] = Time;
u = ns->u;
p = ns->p;
T = ns->T;
gradu = ns->gradu;
dH = ns->dH;
dt = ns->dt; /* direct accses ns */
dt[0] = ns_params->dt0;
ns->gL = gL;
//ns->bk = bk = NULL; //init_line_block(T[1], gL); /* Use line block */
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
/* Init height & depth */
if (gL != NULL){
get_height_depth(ns);
}
/* surf bases */
//surf_bas = ns->surf_bas;
#if 0
DOF *beta = phgDofNew(g, DOF_P1, 1, "beta", func_beta);
phgExportEnsight(g, "check", beta, NULL);
phgFinalize();
exit(1);
#endif
/* ------------------------------------------------------------
*
* Resume dof data.
*
* ------------------------------------------------------------ */
if (ns_params->resume) {
FILE *fp = NULL;
char fname[1000];
phgResumeStage(g, &Time, &tstep, mesh_file, data_file);
/* resume coord */
{
const FLOAT *v = DofData(ns->coord);
int i, k;
//sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", tstep - 1);
sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", 2);
assert(ns->coord->type == DOF_P1);
load_dof_data3(g, ns->coord, data_Crd, mesh_file);
for (i = 0; i < g->nvert; i++)
for (k = 0; k < Dim; k++)
g->verts[i][k] = *(v++);
phgGeomInit_(g, TRUE);
}
#if 1
/* resmue u_{n-1} */
//sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep - 2);
sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 1);
DATA_FILE_SURFIX;
sprintf(fname, "%s.p%03d", data_u, g->rank);
if ((fp = fopen(fname, "r")) == NULL) {
phgPrintf("* u_{%d} unavailable.\n", tstep - 2);
}
else {
fclose(fp);
phgDofCopy(u[1], &u[0], NULL, "u_{n}");
phgDofCopy(p[1], &p[0], NULL, "p_{n}");
phgDofCopy(T[1], &T[0], NULL, "T_{n}");
gradu[0] = NULL;
load_dof_data3(g, u[0], data_u, mesh_file);
load_dof_data3(g, p[0], data_p, mesh_file);
load_dof_data3(g, T[0], data_T, mesh_file);
phgPrintf(" Resume u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n",
tstep - 2, DofGetDataCountGlobal(u[0]), phgDofNormL2(u[0]),
tstep - 2, DofGetDataCountGlobal(p[0]), phgDofNormL2(p[0]));
phgPrintf(" Resume T_{%5d}[%8d]:%24.12E\n",
tstep - 2, DofGetDataCountGlobal(T[0]), phgDofNormL2(T[0]));
phgDofGradient(u[0], &gradu[0], NULL, "gradu_{n}");
phgDofSetFunction(u[0], DofInterpolation);
phgDofSetFunction(p[0], DofInterpolation);
//phgDofSetBdryDataByFunction(u[0], func_u, SETFLOW);
DOF_SCALE(u[0], "resume");
DOF_SCALE(p[0], "resume");
DOF_SCALE(T[0], "resume");
DOF_SCALE(gradu[0], "resume");
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
}
/* resmue u_{n} */
//sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep - 1);
sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 2);
DATA_FILE_SURFIX;
sprintf(fname, "%s.p%03d", data_u, g->rank);
if ((fp = fopen(fname, "r")) == NULL) {
phgError(1, "read Dof data %s failed!\n", data_file);
} else {
fclose(fp);
load_dof_data3(g, u[1], data_u, mesh_file);
load_dof_data3(g, p[1], data_p, mesh_file);
load_dof_data3(g, T[1], data_T, mesh_file);
phgPrintf(" Resume u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n",
tstep - 1, DofGetDataCountGlobal(u[1]), phgDofNormL2(u[1]),
tstep - 1, DofGetDataCountGlobal(p[1]), phgDofNormL2(p[1]));
phgPrintf(" Resume T_{%5d}[%8d]:%24.12E\n",
tstep - 1, DofGetDataCountGlobal(T[1]), phgDofNormL2(T[1]));
phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}");
phgDofSetFunction(u[1], DofInterpolation);
phgDofSetFunction(p[1], DofInterpolation);
//phgDofSetBdryDataByFunction(u[1], func_u, SETFLOW);
DOF_SCALE(u[1], "resume");
DOF_SCALE(p[1], "resume");
DOF_SCALE(T[1], "resume");
DOF_SCALE(gradu[1], "resume");
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
}
#endif
/* Re init height & depth */
if (gL != NULL) {
get_height_depth(ns);
build_layered_mesh_height(g, gL);
check_height(g, gL);
}
/* reconstruct last time step */
//Time -= dt[0];
Time = tstep - 1;
ns->time[1] = Time;
ns->time[0] = Time;
setFuncTime(Time);
#if 0
phgExportEnsightT(g, OUTPUT_DIR "/ins_" NS_PROBLEM , tstep, tstep, u[1], p[1], NULL);
phgFinalize();
return 0;
#endif /* debug exit */
} /* end of resume */
/* Init temp field */
DOF_SCALE(ns->beta, "test");
if (ns_params->solve_temp && tstep == 1) {
phgPrintf("Init temp field!\n");
phgNSTempInit(ns);
}
/* ================================================================================
*
*
* Main loop:
* 1. Steady state: adaptive refinement.
* 2. Time dependent: time advance.
*
* ================================================================================
*/
while (TRUE) {
FLOAT all_memory_usage = phgMemoryUsage(g, NULL)/(1024.0*1024.0);
if (all_memory_usage > 10000)
break;
get_surf_bot_elev(ns);
DOF *grad_surf_elev = phgDofGradient(ns->surf_elev_P1, NULL, NULL, "gradu_surf_elev");
ns->grad_surf_elev = grad_surf_elev;
//phgPrintf("modify mask bot!!\n");
//modify_mask_bot(ns);
//modify_mask_bot(ns);
//phgDofSetDataByValue(u[1], 0.);
//phgDofSetDataByValue(p[1], 0.);
ns->surf_bas = get_surface_bases(g, DOF_P2);
surf_bas = ns->surf_bas;
static BOOLEAN initialized = FALSE;
FLOAT time_end = ns_params->time_end;
elapsed_time(g, FALSE, 0.); /* reset timer */
phgGetTime(tt);
if (Fabs(time_end - Time) < 1e-12) {
phgPrintf("\n=======\nTime reach end: %lf, exit.\n", Time);
phgPrintf("Time End %f\n", time_end);
break;
}
if (tstep > ns_params->max_tstep) {
phgPrintf("\n=======\nTime step reach end: %d, exit.\n",
tstep);
break;
}
#if 0
/* use time t^{n+1} */
dt[-1] = dt[0];
if (Time + dt[0] > time_end)
dt[0] = time_end - Time;
Time += dt[0];
setFuncTime(Time);
#endif
phgPrintf("\n==========\ntime: %lf, step:%d\n", (double)Time, tstep);
phgPrintf(" %d DOF (u:%d, p:%d), %d elements, %d submesh%s, load imbalance: %lg\n",
DofGetDataCountGlobal(u[1]) + DofGetDataCountGlobal(p[1]),
DofGetDataCountGlobal(u[1]), DofGetDataCountGlobal(p[1]),
g->nleaf_global, g->nprocs,
g->nprocs > 1 ? "es" : "", (double)g->lif);
/* save mesh */
if (ns_params->record
&& tstep % ns_params->step_span == 0) {
phgResumeLogUpdate(g, &Time, &tstep, ns_params->fn, NULL);
}
if (!initialized) {
/* reset mem_peak */
phgMemoryUsageReset();
initialized = TRUE;
}
/* ------------------------------------------------------------
*
* Time marching
*
* ------------------------------------------------------------ */
/* update variale,
* call this routine after time update and/or grid change.*/
phgNSTimeAdvance(ns, Time, tstep);
phgPrintf(" update solution");
elapsed_time(g, TRUE, 0.);
if (ns_params->pin_node)
phgNSPinNode(ns);
/* --------------------------------------------------------------------------------
*
* Step 3.
*
* Momentum Equations.
*
* -------------------------------------------------------------------------------- */
int mask_iter = 0, inverse_iter = 0;
INT IF_CHANGE_MASK;
#if 1
while (TRUE)
{
// iteration for ice shelf mask updating and inversion
if ((inverse_iter % 2) == 0)
ns->set_dirichlet_bc = 0;
/* newton bc for the surface, i.e. normal simulation */
else
ns->set_dirichlet_bc = 1;
/* constrain the surface with obs. vel. for inversion */
phgPrintf("----------------------------\n");
phgPrintf("ice shelf mask iteration: %d\n", mask_iter);
phgPrintf("----------------------------\n");
elapsed_time(g, FALSE, 0.); /* reset timer */
/*
* non-linear iteration.
* */
int max_nonstep = 0, newton_start = 0;
assert(ns_params->utype == DOF_P2);
/* For nonlinear iter */
int nonstep = 0;
non_du = non_dp = non_dT = 1e+10;
DOF *u_last = phgDofCopy(u[1], NULL, NULL, "u_last");
DOF *p_last = phgDofCopy(p[1], NULL, NULL, "p_last");
FLOAT non_res_last = 1.;
LTYPE ltype_last = PICARD;
/* First step, change max non step. */
if (tstep == 1) {
if (ns_params->max_nonstep0 > 0)
max_nonstep = ns_params->max_nonstep0;
else
max_nonstep = ns_params->max_nonstep;
if (ns_params->newton_start0 > 0)
newton_start = ns_params->newton_start0;
else
newton_start = ns_params->newton_start;
phgPrintf(" * Set max nonstep to %d for first step.\n", max_nonstep);
phgPrintf(" * Set Newton start to %d for first step.\n", newton_start);
} else {
max_nonstep = ns_params->max_nonstep;
newton_start = ns_params->newton_start;
}
while (TRUE) {
phgPrintf("\n ==================\n");
phgPrintf(" Non-linear interation step: %d\n", nonstep);
/* Init const viscosity */
if (ns_params->start_const_vis &&
tstep == 0 && nonstep == 0 && mask_iter == 0) {
phgPrintf("* vis: const\n");
ns->viscosity_type = VIS_CONST;
} else {
phgPrintf("* vis: strain\n");
ns->viscosity_type = VIS_STRAIN;
}
sayHello("Non linear solve begin");
phgNSInitSolverU(ns);
if (inverse_iter < 1)
get_viscosity(ns);
/* initiate viscosity field at the beginning
* afterwards use the updated viscosity field */
if (nonstep < newton_start)
ns->ltype = PICARD;
else
ns->ltype = NEWTON;
phgPrintf(" Build RHS: ");
phgNSBuildSolverURHS(ns, 1, nonstep, Time);
//phgNSBuildSolverURHS(ns);
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
ns->non_res =
res = phgVecNorm2(ns->solver_u->rhs, 0, NULL);
phgPrintf(" nonlinear residual: %24.12E\n", res);
/* Restore Picard if no improvement */
#if 1
if (ltype_last == NEWTON &&
res > non_res_last * .75) {
phgPrintf(" !!! Newton step failed, use Picard to run again\n");
ns->ltype = PICARD;
max_nonstep += 5; /* Add more Picard steps */
/* resotre dofs:
* Fix me: temprature */
phgDofCopy(u_last, &u[1], NULL, "u_{n+1}");
phgDofCopy(p_last, &p[1], NULL, "p_{n+1}");
phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}");
phgNSBuildSolverURHS(ns, 1, nonstep, Time);
ns->non_res =
res = phgVecNorm2(ns->solver_u->rhs, 0, NULL);
phgPrintf(" nonlinear residual: %24.12E\n", res);
}
#endif
/* save non res */
non_res_last = res;
ltype_last = ns->ltype;
/* build matrices */
//if (ns_params->use_PCD)
//phgNSInitPc(ns);
phgPrintf(" Build matrices:\n");
phgNSBuildSolverUMat(ns, 1, nonstep, Time);
//phgNSBuildSolverUMat(ns);
phgPrintf(" done ");
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
#if 0
if (ns_params->use_PCD) {
phgPrintf(" Build Pc: \n");
phgNSBuildPc(ns);
phgPrintf(" done ");
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
}
#endif
//elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
/*
* solve equation and update (u, p)
* */
phgPrintf("solver tol: %E\n", ns->solver_u->rtol);
phgSolverSolve(ns->solver_u, TRUE, ns->du, ns->dp, NULL);
// elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
//get_water_pressure(ns);
#if USE_NODAL_LOADS
get_nodal_force(ns);
phgMatDestroy(&ns->matF0);
phgMatDestroy(&ns->matB0);
phgMatDestroy(&ns->matBt0);
phgMatDestroy(&ns->matC0);
//phgSolverDestroy(&ns->solver_u0);
ns->solver_u0 = NULL;
#endif
//get_contact(ns);
#if USE_SLIDING_BC
rotate_dof_bases(ns->du, surf_bas, FALSE);
#endif
phgPrintf(" solver_u: nits = %d, resid = %0.4lg ",
ns->solver_u->nits, ns->solver_u->residual);
//elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
/* save dofs */
phgDofCopy(u[1], &u_last, NULL, "u_last");
phgDofCopy(p[1], &p_last, NULL, "p_last");
phgExportVTK(g, "u_test.vtk", ns->u[1], NULL);
/* nonlinear correction */
phgDofAXPY(1.0, ns->du, &u[1]);
phgDofAXPY(1.0, ns->dp, &p[1]);
assert(u[1]->type == ns_params->utype);
assert(p[1]->type == ns_params->ptype);
#if USE_SLIDING_BC
//dof_set_normal_data(u[1], surf_bas);
#else
#endif
/* non_du = phgDofNormL2(ns->du); */
/* non_dp = phgDofNormL2(ns->dp); */
non_du = phgDofNormInftyVec(ns->du);
non_dp = phgDofNormInftyVec(ns->dp);
phgPrintf(" \n du: %24.12E dp: %24.12E\n", non_du, non_dp);
phgPrintf(" u: [%24.12E, %24.12E]\n",
phgDofMinValVec(u[1]),
phgDofMaxValVec(u[1]));
phgPrintf(" p: [%24.12E, %24.12E]\n",
phgDofMinValVec(p[1]),
phgDofMaxValVec(p[1]));
phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}");
//get_avg_gu(ns);
//gradu[1] = ns->avg_gu;
//elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
//if (ns_params->use_PCD)
//phgNSDestroyPc(ns);
//
/* evolution of u */
//DOF_SCALE(u[1], "after solve");
//DOF_SCALE(p[1], "after solve");
#if 0
# warning check solution U,p
sprintf(vtk_file, OUTPUT_DIR "non_%02d_u.vtk", nonstep);
phgExportVTK(g, vtk_file, u[1], p[1], NULL);
phgExportEnsightT(g, OUTPUT_DIR "ins_" NS_PROBLEM ,
nonstep, nonstep, u[1], p[1], T[1],
ns->du, ns->dp, ns->dT, NULL);
#endif
#if 0
if (FALSE && nonstep % ns_params->step_span == 0) {
phgPrintf(" Output solution to ensight ");
phgExportEnsightT(g, OUTPUT_DIR "/ins_" NS_PROBLEM , nonstep, nonstep,
u[1], p[1], T[1], NULL); /* ensight */
sprintf(vtk_file, OUTPUT_DIR "non_%02d_T.vtk", nonstep);
phgExportVTK(g, vtk_file ,
u[1], p[1], T[1], ns->du, NULL);
elapsed_time(g, TRUE, 0.);
//ice_monitor(ns, nonstep);
}
#endif
/* Linearized */
#if 0
if (!ns_params->non_linear
&& nonstep >= 0) {
phgPrintf(" Linearized iteration converges.\n");
break;
}
#endif
phgGetTime(tt1);
phgPrintf(" time usage of current non step: %lfs\n",
(double)(tt1[2] - tt[2]));
nonstep++;
/*
* Nonliner iteration break,
* converge for characteristic value.
* Velocity: 100 m/a
* Pressure: 1e8 Pa
*
* */
INT u_convergence;
if (mask_iter < 1)
u_convergence = check_u_convergence0(ns, u[1], p[1], u_last, p_last, ns_params->u_tol0, ns_params->u_tol0);
else
u_convergence = check_u_convergence(ns, u[1], p[1], u_last, p_last, ns_params->u_tol, ns_params->u_tol);
//elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
const FLOAT U0 = 100;
const FLOAT P0 = 1e8;
if ( nonstep >= ns_params->min_nonstep
&& ns->viscosity_type != VIS_CONST
&& u_convergence
|| nonstep > ns_params->max_nonstep)
{
if (nonstep > ns_params->max_nonstep)
{
phgPrintf(" Non-linear iteration reach max step,"
" results may be inaccrate!\n");
#if 1 && USE_NODAL_LOADS
/*
get_nodal_force(ns);
phgMatDestroy(&ns->matF0);
phgMatDestroy(&ns->matB0);
phgMatDestroy(&ns->matBt0);
phgMatDestroy(&ns->matC0);
//phgSolverDestroy(&ns->solver_u0);
ns->solver_u0 = NULL;
*/
phgMatDestroy(&ns->matF);
phgMatDestroy(&ns->matB);
phgMatDestroy(&ns->matBt);
phgMatDestroy(&ns->matC);
phgSolverDestroy(&ns->solver_u);
ns->solver_u = NULL;
phgMapDestroy(&ns->Vmap);
phgMapDestroy(&ns->Pmap);
phgDofFree(&ns->u_shape);
phgDofFree(&ns->p_shape);
#endif
break;
}
else
{
phgPrintf(" Non-linear iteration converges.\n");
#if 1 && USE_NODAL_LOADS
/*
get_nodal_force(ns);
phgMatDestroy(&ns->matF0);
phgMatDestroy(&ns->matB0);
phgMatDestroy(&ns->matBt0);
phgMatDestroy(&ns->matC0);
//phgSolverDestroy(&ns->solver_u0);
ns->solver_u0 = NULL;
*/
phgMatDestroy(&ns->matF);
phgMatDestroy(&ns->matB);
phgMatDestroy(&ns->matBt);
phgMatDestroy(&ns->matC);
phgSolverDestroy(&ns->solver_u);
ns->solver_u = NULL;
phgMapDestroy(&ns->Vmap);
phgMapDestroy(&ns->Pmap);
phgDofFree(&ns->u_shape);
phgDofFree(&ns->p_shape);
#endif
break;
}
}
phgNSDestroySolverU(ns);
} /* solve */
phgDofFree(&u_last);
phgDofFree(&p_last);
//phgPrintf("Save Dofs\n");
//save_dof_data3(g, u[1], OUTPUT_DIR"u.dat");
//save_dof_data3(g, p[1], OUTPUT_DIR"p.dat");
#elif 0
/* Set velocity, for debugging */
ns->viscosity_type = VIS_STRAIN;
phgDofSetDataByFunction(u[1], func_u0);
//phgDofSetDataByFunction(p[1], func_p0);
phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}");
#else
phgPrintf("Load Dofs\n");
load_dof_data3(g, u[1], OUTPUT_DIR"u.dat", NULL);
load_dof_data3(g, p[1], OUTPUT_DIR"p.dat", NULL);
#endif
/* Project to continuous gradu */
#if 1
phgPrintf("\n ======================= \n");
phgPrintf("project velocity gradient");
phgPrintf("\n======================== \n");
proj_gradu(ns, ns->gradu[1]);
#endif
//DOF_SCALE(gradu[1], "grad u");
//DOF_SCALE(ns->Gradu, "Grad u");
phgPrintf("\n----------------------------\n");
phgPrintf("check ice shelf mask status");
phgPrintf("\n----------------------------\n");
get_mask_bot(ns);
if (!(ns_params->another_run_with_updated_mask))
{
mask_iter = 1;
phgPrintf("manually stops another run with updated mask!\n\n\n");
IF_CHANGE_MASK = if_update_shelf_mask(ns);
IF_CHANGE_MASK = 0;
get_mask_bot(ns);
if (tstep % ns_params->step_span == 0) {
phgPrintf("Save water and nodal forces to VTK \n");
DOF *water_P1 = phgDofCopy(ns->water_force, NULL, DOF_P1, NULL);
DOF *nodal_P1 = phgDofCopy(ns->nodal_force, NULL, DOF_P1, NULL);
sprintf(vtk_file, MASK_OUTPUT_DIR "MASK_%05d.vtk", tstep);
phgExportVTK(g, vtk_file, water_P1,nodal_P1,ns->contact_force,ns->mask_bot,NULL);
phgDofFree(&water_P1);phgDofFree(&nodal_P1);
}
}
if (mask_iter < 1)
{
IF_CHANGE_MASK = if_update_shelf_mask(ns);
}
if ((ns_params->another_run_with_updated_mask)){
if (IF_CHANGE_MASK == 0)
{
phgPrintf("The lower surface mask remains unchanged. Stop the iteration of mask updating !\n");
if (tstep % ns_params->step_span == 0) {
get_mask_bot(ns);
phgPrintf("Save water and nodal forces to VTK \n");
DOF *water_P1 = phgDofCopy(ns->water_force, NULL, DOF_P1, NULL);
DOF *nodal_P1 = phgDofCopy(ns->nodal_force, NULL, DOF_P1, NULL);
sprintf(vtk_file, MASK_OUTPUT_DIR "MASK_%05d.vtk", tstep);
phgExportVTK(g, vtk_file, water_P1,nodal_P1,ns->contact_force,ns->mask_bot,NULL);
phgDofFree(&water_P1);phgDofFree(&nodal_P1);
}
phgDofFree(&ns->nodal_force);
phgDofFree(&ns->water_force);
phgDofFree(&ns->contact_force);
break;
}
}
get_strain_rate(ns);
if (ns->set_dirichlet_bc) {
//DOF *u_d = phgDofCopy(ns->u[1], NULL, DOF_P2, NULL);
DOF *eu_d = phgDofCopy(ns->strain_rate, NULL, DOF_P1, NULL);
ns->eu_d = eu_d;
}
else {
//DOF *u_n = phgDofCopy(ns->u[1], NULL, DOF_P2, NULL);
DOF *eu_n = phgDofCopy(ns->strain_rate, NULL, DOF_P1, NULL);
ns->eu_n = eu_n;
}
DOF *visc_old = phgDofCopy(ns->viscosity, NULL, DOF_P1, NULL);
if (inverse_iter > 0)
update_viscosity_inversion(ns);
FLOAT tol = 0.01;
INT visc_convergence = check_visc_convergence(ns, visc_old, tol);
if((mask_iter >= 1) && (inverse_iter > 0) && (visc_convergence == 1))
{
phgPrintf("\n-------------------------\n");
phgPrintf("ice shelf mask updated \n");
phgPrintf("--------------------------\n");
break;
}
if (mask_iter < 1)
if (tstep % ns_params->step_span == 0) {
phgPrintf("Save water and nodal forces to VTK \n");
DOF *water_P1 = phgDofCopy(ns->water_force, NULL, DOF_P1, NULL);
DOF *nodal_P1 = phgDofCopy(ns->nodal_force, NULL, DOF_P1, NULL);
sprintf(vtk_file, MASK_OUTPUT_DIR "MASK_%05d.vtk", tstep);
phgExportVTK(g, vtk_file, water_P1,nodal_P1,ns->contact_force,ns->mask_bot,NULL);
phgDofFree(&water_P1);phgDofFree(&nodal_P1);
}
phgDofFree(&ns->nodal_force);
phgDofFree(&ns->water_force);
phgDofFree(&ns->contact_force);
//phgDofFree(&ns->mask_bot);
//phgDofFree(&ns->water_pressure);
//phgDofFree(&ns->stress_nn);
// phgPrintf("Free stress_nn !\n");
//phgDofFree(&ns->stress);
//phgDofFree(&ns->water_pressure1);
//phgDofFree(&ns->stress_nn1);
//phgDofFree(&ns->stress1);
//phgDofFree(&ns->avg_gu);
mask_iter++;
inverse_iter++;
}
proj_gradu(ns, ns->gradu[1]);
get_stress(ns, ns->gradu[1], ns->p[1]);
#if 0
if (1) {
int i, k;
DOF *Gu[DDim], *gu[DDim], *guDG0, *stress[DDim];
guDG0 = phgDofCopy(ns->gradu[1], NULL, DOF_P0, NULL);
for (k = 0; k < DDim; k++) {
FLOAT *vGu;
INT n;
char name[1000];
sprintf(name, "Gu%d", k);
Gu[k] = phgDofNew(g, DOF_P1, 1, name, DofNoAction);
vGu = ns->Gradu->data; /* DOF_P1 */
n = DofGetDataCount(Gu[k]);
for (i = 0; i < n; i++)
Gu[k]->data[i] = vGu[i * DDim + k];
sprintf(name, "stress%d", k);
stress[k] = phgDofNew(g, DOF_P1, 1, name, DofNoAction);
vGu = ns->stress->data; /* DOF_P1 */
n = DofGetDataCount(stress[k]);
for (i = 0; i < n; i++)
stress[k]->data[i] = vGu[i * DDim + k];
sprintf(name, "gu%d", k);
gu[k] = phgDofNew(g, DOF_P0, 1, name, DofNoAction);
vGu = guDG0->data; /* DOF_P0 */
n = DofGetDataCount(gu[k]);
for (i = 0; i < n; i++)
gu[k]->data[i] = vGu[i * DDim + k];
}
phgExportVTK(g, "stress.vtk",
stress[0], stress[1], stress[2],
stress[3], stress[4], stress[5],
stress[6], stress[7], stress[8],
NULL);
phgExportVTK(g, "Gu.vtk",
Gu[0], Gu[1], Gu[2],
Gu[3], Gu[4], Gu[5],
Gu[6], Gu[7], Gu[8],
gu[0], gu[1], gu[2],
gu[3], gu[4], gu[5],
gu[6], gu[7], gu[8],
NULL);
// phgFinalize();
}
#endif
/* --------------------------------------------------------------------------------
*
* Step 4.
*
* Solve temperature.
*
* -------------------------------------------------------------------------------- */
#if 1
if (ns_params->solve_temp) {
phgPrintf("\n ==================\n");
phgPrintf(" Temperature solve \n");
phgPrintf(" ==================\n\n");
phgPrintf(" T type: %s\n", T[1]->type->name);
elapsed_time(g, FALSE, 0.); /* reset timer */
phgNSInitSolverT(ns);
phgPrintf(" Build Mat: ");
phgNSBuildSolverTMat(ns, FALSE);
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
phgPrintf(" Build RHS: ");
phgNSBuildSolverTRHS(ns, FALSE);
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
phgNSSolverTBuildConstrain(ns);
phgDofCopy(ns->T[1], &ns->dT, NULL, "dT");
phgNSSolverTSolve(ns, FALSE);
elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL));
phgNSDestroySolverT(ns);
//find_melt_region(ns);
DOF_SCALE(ns->T[1], "after solve");
phgDofAXPY(-1.0, ns->T[1], &ns->dT);
non_dT = phgDofNormInftyVec(ns->dT);
phgPrintf(" dT: %24.12E\n", non_dT);
DOF *temp_diff = phgDofCopy(ns->T[1], NULL, NULL, "Td");
{
FLOAT *vt = temp_diff->data;
const FLOAT *vh = ns->depth_P2->data;
INT i, n = DofGetDataCount(temp_diff);
for (i = 0; i < n; i++, vh++, vt++)
*vt = TEMP_WATER - BETA_MELT * (*vh) *LEN_SCALING - (*vt);
}
} else {
phgPrintf("Temp not updated.\n");
non_dT = 0.;
}
#endif
/* ------------------------------------------------------------
*
* Error check
*
* ------------------------------------------------------------ */
#if 0
if (!ns_params->compute_error) {
eu = ep = egradu = eT = NULL;
ediv = phgDofDivergence(u[1], NULL, NULL, "err div u");
phgPrintf( " normL2(u, p) = (%20.12E, %20.12E)\n"
" normH1(u) = (%20.12E)\n"
" normDiv(u) = (%20.12E)\n"
" normGadu = (%20.12E)\n",
dofNormL2(u[1]), dofNormL2(p[1]),
dofNormL2(gradu[1]), dofNormL2(ediv),
dofNormL2(ns->gradu[1]));
elapsed_time(g, TRUE, 0.);
} else {
/* ----Error check------- */
phgPrintf(" Errors: \n");
eu = phgDofCopy(u[1], NULL, ns_params->utype, "erru");
ep = phgDofCopy(p[1], NULL, ns_params->ptype, "errp");
eT = phgDofCopy(T[1], NULL, NULL, "errT");
egradu = phgDofCopy(gradu[1], NULL, NULL, "err grad u");
ediv = phgDofDivergence(u[1], NULL, NULL, "err div u");
adjust_time(- (1. - ns_params->Theta) * dt[0]);
restore_time();
phgPrintf(" errL2(u, p) = (%20.12E, %20.12E)\n"
" errH1(u) = (%20.12E)\n"
" errDiv(u) = (%20.12E)\n",
dofNormL2(eu), dofNormL2(ep),
dofNormL2(egradu), dofNormL2(ediv));
elapsed_time(g, TRUE, 0.);
phgDofFree(&egradu);
dof_norm_L2(eT);
}
#endif
getPecletNum(g, u[1], ns_params->nu, 6);
mem = phgMemoryUsage(g, &mem_peak);
phgPrintf(" Memory usage: current %0.4lgMB, peak %0.4lgMB\n",
(double)mem / (1024.0 * 1024.0),
(double)mem_peak / (1024.0 * 1024.0));
/* ------------------------------------------------------------
*
* Move mesh
*
* ------------------------------------------------------------ */
FLOAT ice_volume_last = get_ice_volume(g);
// DOF *dH_last = phgDofCopy(ns->dH, NULL, NULL, "dH_last");
if (ns_params->solve_height) {
if (gL != NULL) {
/* Unstructed layered mesh */
phgPrintf("Move mesh.\n");
get_surf_dH(ns);
//DOF *dH_fem = phgDofCopy(ns->dH, NULL, NULL, NULL);
phgExportVTK(g, "dH_fem1.vtk", ns->dH, NULL);
get_smooth_surface_values(ns, ns->dH, 0);
get_smooth_surface_values(ns, ns->dH, 1);
phgExportVTK(g, "dH_fem.vtk", ns->dH, NULL);
/*
save_free_surface_elev(ns, 0);
save_free_surface_elev(ns, 1);
save_free_surface_velo(ns, 0, 0);
save_free_surface_velo(ns, 1, 0);
save_free_surface_velo(ns, 2, 0);
save_free_surface_velo(ns, 0, 1);
save_free_surface_velo(ns, 1, 1);
save_free_surface_velo(ns, 2, 1);
if (phgRank == 0)
{
system("python get_upper_ds.py");
system("python get_lower_ds.py");
}
load_dH_from_file(ns, ns->dH, 0);
load_dH_from_file(ns, ns->dH, 1);
//DOF *dH_fdm = phgDofCopy(ns->dH, NULL, NULL, NULL);
//phgDofAXPY(-1, dH_fem, &dH_fdm);
//phgExportVTK(g, "dH_diff.vtk", dH_fdm, NULL);
*///phgExportVTK(g, "dH_fdm.vtk", ns->dH, NULL);
get_moved_coord(ns, tstep);
phgExportVTK(g, "dH_fem2.vtk", NULL);
move_mesh(ns);
//phgExportVTK(g, "moved_geo.vtk", NULL);
//check_height(g, gL);
} else {
/* Structed layered mesh */
struct_mesh_update(ns, tstep, Time);
}
phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}");
phgDofGradient(u[0], &gradu[0], NULL, "gradu_{n}");
} else {
phgPrintf("Mesh not moved.\n");
}
phgPrintf("\n-----------------------------------------\n");
phgPrintf(" dH: [%4.2E, %4.2E]\n",
phgDofMinValVec(ns->dH),
phgDofMaxValVec(ns->dH));
phgPrintf("------------------------------------------\n\n");
FLOAT ice_volume = get_ice_volume(g);
//DOF *u_P1 = phgDofCopy(u[1], NULL, DOF_P1, NULL);
//phgExportVTK(g, "u_P1.vtk", u_P1, NULL);
#if 1
//INT dH_convergence = check_surf_convergence(ns, dH_last);
//if (dH_convergence == 1)
FLOAT dVdt = fabs(ice_volume-ice_volume_last)/ice_volume/dt[0];
if (Time > 10 && dVdt < ns_params->s_tol)
{
phgPrintf("-----------------------------------------------------\n");
phgPrintf("The ice domain reaches a steady state! Model stops! dVdt %e", dVdt);
phgPrintf("\n-----------------------------------------------------\n");
break;
}
else
{
phgPrintf("\n--------------------------------------------------------------\n");
phgPrintf("The ice domain is still in an unsteady state! Model continues! dVdt %e\n", dVdt);
phgPrintf("---------------------------------------------------------------\n\n");
}
#endif
update_grounding_line(ns, tstep);
// after we update the geometry, we need to check the ice shelf mask again
/* ------------------------------------------------------------
*
* Output solution
*
* ------------------------------------------------------------ */
if (tstep % ns_params->step_span == 0) {
//ice_monitor(ns, tstep);
#if 1
/* Temp check */
phgPrintf(" Output solution to VTK");
/* phgExportEnsightT(g, OUTPUT_DIR "/ins_" NS_PROBLEM , Time, tstep, */
/* u[1], p[1], T[1], ns->depth_P1, */
/* NULL); /\* ensight *\/ */
DOF *u_P1 = phgDofCopy(u[1], NULL, DOF_P1, NULL);
//phgExportVTK(g,"results.vtk",u_P1,p[1],NULL);
sprintf(vtk_file, dH_OUTPUT_DIR "ice_%05d.vtk", tstep);
phgExportVTK(g, vtk_file, u_P1, NULL);
phgDofFree(&u_P1);
sprintf(vtk_file, dH_OUTPUT_DIR "dH_%05d.vtk", tstep);
phgExportVTK(g, vtk_file, ns->dH, NULL);
sprintf(vtk_file, dH_OUTPUT_DIR "stress_%05d.vtk", tstep);
phgExportVTK(g, vtk_file, ns->stress, NULL);
//sprintf(vtk_file, OUTPUT_DIR "mask_%05d.vtk", tstep);
//phgExportVTK(g, vtk_file, ns->mask_bot, NULL);
elapsed_time(g, TRUE, 0.);
#endif
/* Save coord data */
#if 1
if (tstep % ns_params->step_span_resume == 0) {
{
//sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", tstep);
sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", 2);
assert(ns->coord->type == DOF_P1);
save_dof_data3(g, ns->coord, data_Crd);
}
if (ns_params->record) {
/* save dof data for time step {n}, {n+1} */
//sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep - 1);
sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 1);
DATA_FILE_SURFIX;
save_dof_data3(g, u[0], data_u);
save_dof_data3(g, p[0], data_p);
save_dof_data3(g, T[0], data_T);
phgPrintf(" Save u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n",
tstep - 1, DofGetDataCountGlobal(u[0]), phgDofNormL2(u[0]),
tstep - 1, DofGetDataCountGlobal(p[0]), phgDofNormL2(p[0]));
phgPrintf(" Save T_{%5d}[%8d]:%24.12E\n",
tstep - 1, DofGetDataCountGlobal(T[0]), phgDofNormL2(T[0]));
DOF_SCALE(u[0], "save");
DOF_SCALE(p[0], "save");
DOF_SCALE(T[0], "save");
DOF_SCALE(gradu[0], "save");
//sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep);
sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 2);
DATA_FILE_SURFIX;
save_dof_data3(g, u[1], data_u);
save_dof_data3(g, p[1], data_p);
save_dof_data3(g, T[1], data_T);
phgPrintf(" Save u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n",
tstep, DofGetDataCountGlobal(u[1]), phgDofNormL2(u[1]),
tstep, DofGetDataCountGlobal(p[1]), phgDofNormL2(p[1]));
phgPrintf(" Save T_{%5d}[%8d]:%24.12E\n",
tstep, DofGetDataCountGlobal(T[1]), phgDofNormL2(T[1]));
DOF_SCALE(u[1], "save");
DOF_SCALE(p[1], "save");
DOF_SCALE(T[1], "save");
DOF_SCALE(gradu[1], "save");
phgResumeLogUpdate(g, NULL, NULL, NULL, data_file);
} /* end of record */
if (gL != NULL) {
//check_height(g, gL);
}
sayHello("After record final solution data");
}
#endif
}
//phgDofFree(&dH_last);
/* clean up */
//phgDofFree(&ediv);
//phgDofFree(&eu);
//phgDofFree(&ep);
//phgDofFree(&eT);
/* ----------------------------------------------------------------------
*
* Compute drag force FD
*
* ----------------------------------------------------------------------
* */
phgGetTime(tt1);
phgPrintf(" total time usage of current time step: %lfs\n",
(double)(tt1[2] - tt[2]));
if (mem_peak > 1024 * (size_t)ns_params->mem_max * 1024) {
phgPrintf("\n=======\nMem usage reach max, exit.\n");
break;
}
tstep++;
#if 1
/* use time t^{n} */
dt[-1] = dt[0];
if (Time + dt[0] > time_end)
dt[0] = time_end - Time;
Time += dt[0];
setFuncTime(Time);
#endif
phgDofFree(&surf_bas->dof);
phgDofFree(&ns->surf_bas->dof);
} /* end of time advaning */
/* destroy line block */
//destroy_line_block(&ns->bk);
/* destroy reused solver */
if (ns->solver_u != NULL) {
if (ns_params->use_PCD)
phgNSDestroyPc(ns);
phgNSDestroySolverU(ns);
}
phgNSFinalize(&ns);
phgFreeGrid(&g);
phgFinalize();
phgFree(ns_params);
return 0;
}
int
main(int argc, char *argv[])
{
MAT *A, *B;
VEC *U = NULL, *x;
FLOAT *C;
SOLVER *solver, *solver1 = NULL;
INT i, j, k, n, *pvt, N = 1000, K = 2;
char *main_opts = NULL, *sub_opts = NULL;
phgOptionsRegisterInt("-n", "N value", &N);
phgOptionsRegisterInt("-k", "K value", &K);
phgOptionsRegisterString("-main_solver_opts", "Options for the main solver",
&main_opts);
phgOptionsRegisterString("-sub_solver_opts", "Options for the subsolver",
&sub_opts);
/* a direct solver is preferable for the sparse matrix */
phgOptionsPreset("-solver mumps");
phgInit(&argc, &argv);
phgPrintf(
"----------------------------------------------------------------------------\n"
"This code solves (A+UU^t)x=b using the Sherman-Morrison-Woodbury formula.\n"
"Note: may use the following to disable use of the Sherman-Morrison-Woodbury\n"
"algorithm and change to the default solver instead:\n"
" -preonly_pc_type solver -preonly_pc_opts \"-solver_maxit 2000\"\n"
"----------------------------------------------------------------------------\n"
);
phgPrintf("Generating the linear system: N = %"dFMT", K = %"dFMT"\n", N, K);
/* A is a distributed NxN SPD tridiagonal matrix (A = [-1, 2, -1]) */
n = N / phgNProcs + (phgRank < (N % phgNProcs) ? 1 : 0);
A = phgMatCreate(phgComm, n, N);
phgPrintf(" Generating matrix A.\n");
for (i = 0; i < n; i++) {
/* diagonal */
phgMatAddEntry(A, i, i, 2.0);
/* diagonal - 1 */
if (i > 0)
phgMatAddEntry(A, i, i - 1, -1.0);
else if (phgRank > 0)
phgMatAddLGEntry(A, i, A->rmap->partition[phgRank] - 1, -1.0);
/* diagonal + 1 */
if (i < n - 1)
phgMatAddEntry(A, i, i + 1, -1.0);
else if (phgRank < phgNProcs - 1)
phgMatAddLGEntry(A, i, A->rmap->partition[phgRank] + n, -1.0);
}
phgMatAssemble(A);
/* U is a K-component vector */
U = phgMapCreateVec(A->rmap, K);
phgVecRandomize(U, 123);
/* solver1 is the solver for A */
phgOptionsPush();
phgOptionsSetOptions(sub_opts);
solver1 = phgMat2Solver(SOLVER_DEFAULT, A);
phgOptionsPop();
/* x is a scratch vector */
x = phgMapCreateVec(A->rmap, 1);
/* C is a KxK dense matrix, pvt is an integer array, they store the LU
* factorization of (I + U^t*inv(A)*U) */
phgPrintf(" Generating the dense matrix I+U^t*inv(A)*U.\n");
C = phgCalloc(K * K, sizeof(*C));
pvt = phgAlloc(K * sizeof(*pvt));
for (i = 0; i < K; i++) {
for (j = 0; j < n; j++) {
solver1->rhs->data[j] = U->data[i * n + j];
x->data[j] = 0.0;
}
solver1->rhs->assembled = TRUE;
phgSolverVecSolve(solver1, FALSE, x);
for (j = 0; j < K; j++)
for (k = 0; k < n; k++)
C[i * K + j] += U->data[j * n + k] * x->data[k];
}
#if USE_MPI
if (U->map->nprocs > 1) {
FLOAT *tmp = phgAlloc(K * K * sizeof(*tmp));
MPI_Allreduce(C, tmp, K * K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm);
phgFree(C);
C = tmp;
}
#endif /* USE_MPI */
for (i = 0; i < K; i++)
C[i * K + i] += 1.0;
phgPrintf(" Factorizing the dense matrix I+U^t*inv(A)*U.\n");
phgSolverDenseLU(K, C, pvt);
/* B is a matrix-free matrix representing A + U*U^t, B->mv_data is used
* to pass A, U, solver1, C and pvt to callback functions */
B = phgMapCreateMatrixFreeMat(A->rmap, A->cmap, funcB,
/* arguments carried over to CB functions */
A, U, solver1, C, pvt, NULL);
/* solver is a PreOnly solver for B whose pc_proc is set to sherman().
*
* Note: can also use pcg, gmres, or petsc for this solver, in this case
* the solution obtained with the Sherman-Morisson formula is iteratively
* refined. */
phgOptionsPush();
phgOptionsSetOptions("-solver preonly");
phgOptionsSetOptions(main_opts);
solver = phgMat2Solver(SOLVER_DEFAULT, B);
phgSolverSetPC(solver, solver, sherman);
phgOptionsPop();
for (i = 0; i < n; i++)
x->data[i] = 1.0;
phgMatVec(MAT_OP_N, 1.0, B, x, 0.0, &solver->rhs);
phgPrintf("Solving the linear system.\n");
/* reset initial solution to zero */
memset(x->data, 0, n * sizeof(*x->data));
phgSolverVecSolve(solver, TRUE, x);
for (i = 0; i < n; i++)
solver->rhs->data[i] = 1.0;
phgVecAXPBY(-1.0, solver->rhs, 1.0, &x);
phgPrintf("Checking the result: |x - x_exact| / |x_exact| = %lg\n",
(double)phgVecNorm2(x, 0, NULL) / sqrt((double)N));
phgSolverDestroy(&solver);
phgSolverDestroy(&solver1);
phgMatDestroy(&A);
phgMatDestroy(&B);
phgVecDestroy(&U);
phgVecDestroy(&x);
phgFree(C);
phgFree(pvt);
phgFinalize();
return 0;
}
int
main(int argc, char *argv[])
{
GRID *g;
DOF *u_h;
MAT *A, *A0, *B;
MAP *map;
INT i;
size_t nnz, mem, mem_peak;
VEC *x, *y0, *y1, *y2;
double t0, t1, dnz, dnz1, mflops, mop;
char *fn = "../test/cube.dat";
FLOAT mem_max = 300;
INT refine = 0;
phgOptionsRegisterFilename("-mesh_file", "Mesh file", (char **)&fn);
phgOptionsRegisterInt("-loop_count", "Loop count", &loop_count);
phgOptionsRegisterInt("-refine", "Refinement level", &refine);
phgOptionsRegisterFloat("-mem_max", "Maximum memory", &mem_max);
phgInit(&argc, &argv);
g = phgNewGrid(-1);
if (!phgImport(g, fn, FALSE))
phgError(1, "can't read file \"%s\".\n", fn);
phgRefineAllElements(g, refine);
u_h = phgDofNew(g, DOF_DEFAULT, 1, "u_h", DofNoAction);
while (TRUE) {
phgPrintf("\n");
if (phgBalanceGrid(g, 1.2, 1, NULL, 0.))
phgPrintf("Repartition mesh, %d submeshes, load imbalance: %lg\n",
g->nprocs, (double)g->lif);
map = phgMapCreate(u_h, NULL);
A = phgMapCreateMat(map, map);
A->handle_bdry_eqns = TRUE;
build_matrix(A, u_h);
phgMatAssemble(A);
/* Note: A is unsymmetric (A' != A) if boundary entries not removed */
phgMatRemoveBoundaryEntries(A);
#if 0
/* test block matrix operation */
A0 = phgMatCreateBlockMatrix(g->comm, 1, 1, &A, NULL);
#else
A0 = A;
#endif
phgPrintf("%d DOF, %d elems, %d submeshes, matrix size: %d, LIF: %lg\n",
DofGetDataCountGlobal(u_h), g->nleaf_global,
g->nprocs, A->rmap->nglobal, (double)g->lif);
/* test PHG mat-vec multiply */
x = phgMapCreateVec(A->cmap, 1);
y1 = phgMapCreateVec(A->rmap, 1);
phgVecRandomize(x, 123);
phgMatVec(MAT_OP_N, 1.0, A0, x, 0.0, &y1);
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
phgMatVec(MAT_OP_N, 1.0, A0, x, 0.0, &y1);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
y0 = phgVecCopy(y1, NULL);
nnz = A->nnz_d + A->nnz_o;
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
mop = loop_count * (dnz + dnz - A->rmap->nlocal) * 1e-6;
phgPrintf("\n");
t1 -= t0;
phgPrintf(" PHG: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF)\n",
t1, dnz, mop / (t1 == 0 ? 1. : t1), mflops);
/* test trans(A)*x */
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
phgMatVec(MAT_OP_T, 1.0, A0, x, 0.0, &y1);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
t1 -= t0;
phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1 == 0 ? 1. : t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
/* time A * trans(A) */
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
B = phgMatMat(MAT_OP_N, MAT_OP_N, 1.0, A, A, 0.0, NULL);
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
nnz = B->nnz_d + B->nnz_o;
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
/* compare B*x <--> A*A*x */
y2 = phgMatVec(MAT_OP_N, 1.0, B, x, 0.0, NULL);
phgMatVec(MAT_OP_N, 1.0, A0, y0, 0.0, &y1);
phgMatDestroy(&B);
t1 -= t0;
phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n",
t1, dnz, mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL));
#if USE_PETSC
{
Mat ma, mb;
MatInfo info;
Vec va, vb, vc;
PetscScalar *vec;
ma = phgPetscCreateMatAIJ(A);
MatGetVecs(ma, PETSC_NULL, &va);
VecDuplicate(va, &vb);
VecGetArray(va, &vec);
memcpy(vec, x->data, x->map->nlocal * sizeof(*vec));
VecRestoreArray(va, &vec);
MatMult(ma, va, vb);
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
MatMult(ma, va, vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
VecGetArray(vb, &vec);
memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vb, &vec);
MatGetInfo(ma, MAT_GLOBAL_SUM, &info);
/*phgPrintf(" --------------------------------------------"
"-------------------------\n");*/
phgPrintf("\n");
t1 -= t0;
dnz = info.nz_used;
phgPrintf(" PETSc: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
MatMultTranspose(ma, va, vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
VecGetArray(vb, &vec);
memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vb, &vec);
t1 -= t0;
phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
MatMatMult(ma, ma, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &mb);
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
t1 -= t0;
MatGetInfo(mb, MAT_GLOBAL_SUM, &info);
dnz = info.nz_used;
VecDuplicate(va, &vc);
/* compare B*x <--> A*A*x */
MatMult(ma, vb, vc);
MatMult(mb, va, vb);
VecGetArray(vb, &vec);
memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vb, &vec);
VecGetArray(vc, &vec);
memcpy(y2->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vc, &vec);
phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n",
t1, dnz, mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL));
phgPetscMatDestroy(&mb);
phgPetscMatDestroy(&ma);
phgPetscVecDestroy(&va);
phgPetscVecDestroy(&vb);
phgPetscVecDestroy(&vc);
}
#endif /* USE_PETSC */
#if USE_HYPRE
{
HYPRE_IJMatrix ma;
HYPRE_IJVector va, vb, vc;
HYPRE_ParCSRMatrix par_ma;
hypre_ParCSRMatrix *par_mb;
HYPRE_ParVector par_va, par_vb, par_vc;
HYPRE_Int offset, *ni, start, end;
assert(sizeof(INT)==sizeof(int) && sizeof(FLOAT)==sizeof(double));
setup_hypre_mat(A, &ma);
ni = phgAlloc(2 * A->rmap->nlocal * sizeof(*ni));
offset = A->cmap->partition[A->cmap->rank];
for (i = 0; i < A->rmap->nlocal; i++)
ni[i] = i + offset;
HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1,
&va);
HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1,
&vb);
HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1,
&vc);
HYPRE_IJVectorSetObjectType(va, HYPRE_PARCSR);
HYPRE_IJVectorSetObjectType(vb, HYPRE_PARCSR);
HYPRE_IJVectorSetObjectType(vc, HYPRE_PARCSR);
HYPRE_IJVectorSetMaxOffProcElmts(va, 0);
HYPRE_IJVectorSetMaxOffProcElmts(vb, 0);
HYPRE_IJVectorSetMaxOffProcElmts(vc, 0);
HYPRE_IJVectorInitialize(va);
HYPRE_IJVectorInitialize(vb);
HYPRE_IJVectorInitialize(vc);
HYPRE_IJMatrixGetObject(ma, (void **)(void *)&par_ma);
HYPRE_IJVectorGetObject(va, (void **)(void *)&par_va);
HYPRE_IJVectorGetObject(vb, (void **)(void *)&par_vb);
HYPRE_IJVectorGetObject(vc, (void **)(void *)&par_vc);
HYPRE_IJVectorSetValues(va, A->cmap->nlocal, ni, (double *)x->data);
HYPRE_IJVectorAssemble(va);
HYPRE_IJVectorAssemble(vb);
HYPRE_IJVectorAssemble(vc);
HYPRE_IJMatrixGetRowCounts(ma, A->cmap->nlocal,
ni, ni + A->rmap->nlocal);
for (i = 0, nnz = 0; i < A->rmap->nlocal; i++)
nnz += ni[A->rmap->nlocal + i];
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_va, 0.0, par_vb);
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_va, 0.0, par_vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data);
/*phgPrintf(" --------------------------------------------"
"-------------------------\n");*/
phgPrintf("\n");
t1 -= t0;
phgPrintf(" HYPRE: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
HYPRE_ParCSRMatrixMatvecT(1.0, par_ma, par_va, 0.0, par_vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data);
t1 -= t0;
phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
/* Note: 'HYPRE_ParCSRMatrix' is currently typedef'ed to
* 'hypre_ParCSRMatrix *' */
par_mb = hypre_ParMatmul((hypre_ParCSRMatrix *)par_ma,
(hypre_ParCSRMatrix *)par_ma);
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
start = hypre_ParCSRMatrixFirstRowIndex(par_mb);
end = hypre_ParCSRMatrixLastRowIndex(par_mb) + 1;
for (i = start, nnz = 0; i < end; i++) {
HYPRE_Int ncols;
hypre_ParCSRMatrixGetRow(par_mb, i, &ncols, NULL, NULL);
hypre_ParCSRMatrixRestoreRow(par_mb, i, &ncols, NULL, NULL);
nnz += ncols;
}
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
/* compare B*x <--> A*A*x */
HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_vb, 0.0, par_vc);
HYPRE_ParCSRMatrixMatvec(1.0, (void *)par_mb, par_va, 0.0, par_vb);
HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data);
HYPRE_IJVectorGetValues(vc, A->rmap->nlocal, ni, (double*)y2->data);
hypre_ParCSRMatrixDestroy((par_mb));
t1 -= t0;
phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n",
t1, dnz, mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL));
phgFree(ni);
HYPRE_IJMatrixDestroy(ma);
HYPRE_IJVectorDestroy(va);
HYPRE_IJVectorDestroy(vb);
HYPRE_IJVectorDestroy(vc);
}
#endif /* USE_HYPRE */
if (A0 != A)
phgMatDestroy(&A0);
#if 0
if (A->rmap->nglobal > 1000) {
VEC *v = phgMapCreateVec(A->rmap, 3);
for (i = 0; i < v->map->nlocal; i++) {
v->data[i + 0 * v->map->nlocal] = 1 * (i + v->map->partition[g->rank]);
v->data[i + 1 * v->map->nlocal] = 2 * (i + v->map->partition[g->rank]);
v->data[i + 2 * v->map->nlocal] = 3 * (i + v->map->partition[g->rank]);
}
phgMatDumpMATLAB(A, "A", "A.m");
phgVecDumpMATLAB(v, "v", "v.m");
phgFinalize();
exit(0);
}
#endif
phgMatDestroy(&A);
phgVecDestroy(&x);
phgVecDestroy(&y0);
phgVecDestroy(&y1);
phgVecDestroy(&y2);
phgMapDestroy(&map);
mem = phgMemoryUsage(g, &mem_peak);
dnz = mem / (1024.0 * 1024.0);
dnz1 = mem_peak / (1024.0 * 1024.0);
/*phgPrintf(" --------------------------------------------"
"-------------------------\n");*/
phgPrintf("\n");
phgPrintf(" Memory: current %0.4lgMB, peak %0.4lgMB\n", dnz, dnz1);
#if 0
{
static int loop_count = 0;
if (++loop_count == 4)
break;
}
#endif
if (mem_peak > 1024 * (size_t)1024 * mem_max)
break;
phgRefineAllElements(g, 1);
}
phgDofFree(&u_h);
phgFreeGrid(&g);
phgFinalize();
return 0;
}
