/* PC_PROC function which applies the Sherman-Morrison-Woodbury formula */ static void sherman(void *ctx, VEC *b0, VEC **x0) { SOLVER *solver = ctx; VEC *U = ((void **)solver->mat->mv_data)[1]; SOLVER *solver1 = ((void **)solver->mat->mv_data)[2]; FLOAT *C = ((void **)solver->mat->mv_data)[3]; INT *pvt = ((void **)solver->mat->mv_data)[4]; VEC *x = *x0, *y; INT i, j, n = U->map->nlocal, K = U->nvec; FLOAT *z; phgPrintf("Note: applying the Sherman-Morrison-Woodbury formula.\n"); /* x := inv(A) * b */ #if 0 memset(x->data, 0, n * sizeof(*x->data)); #endif z = solver1->rhs->data; /* save solver1->rhs->data */ solver1->rhs->data = b0->data; solver1->rhs->assembled = TRUE; phgSolverVecSolve(solver1, FALSE, x); solver1->rhs->data = z; /* restore solver1->rhs->data */ if (K == 0) return; /* z := U^t * x */ z = phgCalloc(K, sizeof(*z)); for (j = 0; j < K; j++) for (i = 0; i < n; i++) z[j] += U->data[j * n + i] * x->data[i]; #if USE_MPI if (U->map->nprocs > 1) { FLOAT *tmp = phgAlloc(K * sizeof(*tmp)); MPI_Allreduce(z, tmp, K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm); phgFree(z); z = tmp; } #endif /* USE_MPI */ /* z := C*z */ phgSolverDenseSV(K, C, pvt, 1, z); /* y := inv(A) * (U*z) */ memset(solver1->rhs->data, 0, n * sizeof(*solver1->rhs->data)); for (i = 0; i < n; i++) for (j = 0; j < K; j++) solver1->rhs->data[i] += U->data[j * n + i] * z[j]; y = phgMapCreateVec(U->map, 1); phgSolverVecSolve(solver1, FALSE, y); phgFree(z); /* x -= y */ for (i = 0; i < n; i++) x->data[i] -= y->data[i]; phgVecDestroy(&y); }
static void do_test(MAP *rmap, MAP *cmap) { MAT *A; VEC *x, *y; INT i, j, m0, n0; INT *cols = phgAlloc(cmap->nglobal * sizeof(*cols)); FLOAT *data = phgAlloc(cmap->nglobal * sizeof(*data)); A = phgMapCreateMat(rmap, cmap); m0 = A->rmap->partition[A->rmap->rank]; n0 = A->cmap->partition[A->cmap->rank]; /* Matrix entries: A(I,J) = 1 + (I-1) + (J-1) */ for (i = m0; i < m0 + A->rmap->nlocal; i++) { #if 0 /* Test MatAddEntry */ for (j = 0; j < A->cmap->nglobal; j++) phgMatAddGlobalEntry(A, i, j, 1.0 + i + j); #else /* Test MatAddEntries */ for (j = 0; j < A->cmap->nglobal; j++) { cols[j] = j; data[j] = 1.0 + i + j; } phgMatAddGlobalEntries(A, 1, &i, A->cmap->nglobal, cols, data); #endif } phgFree(cols); phgFree(data); phgMatAssemble(A); phgInfo(-1, "y = A * x\n"); x = phgMapCreateVec(A->cmap, 1); for (i = 0; i < x->map->nlocal; i++) x->data[i] = 1.0 + i + n0; phgVecAssemble(x); y = phgMatVec(MAT_OP_N, 1.0, A, x, 0.0, NULL); for (i = 0; i < y->map->nlocal; i++) phgInfo(-1, " y->data[%d] = %lg\n", i + m0, (double)y->data[i]); phgVecDestroy(&x); phgVecDestroy(&y); phgInfo(-1, "y = A' * x\n"); x = phgMapCreateVec(A->rmap, 1); for (i = 0; i < x->map->nlocal; i++) x->data[i] = 1.0 + i + m0; phgVecAssemble(x); y = phgMatVec(MAT_OP_T, 1.0, A, x, 0.0, NULL); for (i = 0; i < y->map->nlocal; i++) phgInfo(-1, " y->data[%d] = %lg\n", i + n0, (double)y->data[i]); phgVecDestroy(&x); phgVecDestroy(&y); phgMatDestroy(&A); }
/* callback function which computes A+UU^t */ static int funcB(MAT_OP op, MAT *B, VEC *x, VEC *y) /* computes y = op(A) * x */ { MAT *A = ((void **)B->mv_data)[0]; VEC *U = ((void **)B->mv_data)[1]; INT i, j, k, n = U->map->nlocal, K = U->nvec; /* compute y = op(A)*x */ phgMatVec(op, 1.0, A, x, 0.0, &y); if (K == 0) return 0; if (op != MAT_OP_D) { /* compute y += U*U^t*x */ FLOAT *z = phgCalloc(K, sizeof(*z)); /* z = U^t*x */ for (j = 0; j < K; j++) for (i = 0; i < n; i++) z[j] += U->data[j * n + i] * x->data[i]; #if USE_MPI if (U->map->nprocs > 1) { FLOAT *tmp = phgCalloc(K, sizeof(*tmp)); MPI_Allreduce(z, tmp, K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm); phgFree(z); z = tmp; } #endif /* USE_MPI */ /* y += U*z */ for (i = 0; i < n; i++) for (j = 0; j < K; j++) y->data[i] += U->data[j * n + i] * z[j]; phgFree(z); return 0; } for (i = 0; i < n; i++) { FLOAT d = 0.0; for (j = 0; j < K; j++) for (k = 0; k < K; k++) d += U->data[j * n + i] * U->data[k * n + i]; y->data[i] += d * x->data[i]; } return 0; }
static void setup_hypre_mat(MAT *A, HYPRE_IJMatrix *pa) { MAT_ROW *row; INT i, j, d, I, n0, n, *cols; assert(sizeof(INT) == sizeof(int) && sizeof(FLOAT)==sizeof(double)); phgMatUnpack(A); HYPRE_IJMatrixCreate(A->rmap->comm, A->rmap->partition[A->rmap->rank], A->rmap->partition[A->rmap->rank + 1] - 1, A->cmap->partition[A->cmap->rank], A->cmap->partition[A->cmap->rank + 1] - 1, pa); HYPRE_IJMatrixSetObjectType(*pa, HYPRE_PARCSR); HYPRE_IJMatrixInitialize(*pa); I = A->rmap->partition[A->rmap->rank]; n0 = A->cmap->partition[A->cmap->rank]; n = 0; cols = NULL; for (i = 0, row = A->rows; i < A->rmap->nlocal; i++, row++, I++) { if (row->ncols > 0) { if (n < row->ncols) { phgFree(cols); cols = phgAlloc((n = row->ncols) * sizeof(*cols)); } for (j = 0; j < row->ncols; j++) { if ((d = row->cols[j]) < A->cmap->nlocal) cols[j] = d + n0; else cols[j] = A->O2Gmap[d - A->cmap->nlocal]; } HYPRE_IJMatrixSetValues(*pa, 1, &row->ncols, &I, cols, row->data); } } phgFree(cols); HYPRE_IJMatrixAssemble(*pa); }
/* * Jacobi smoother2: * Implement using details of matvec * ref: matvec in matvec.c * * */ void mg_Jacobi2(MAT *A, VEC *x, VEC *b, int nsmooth, void *ctx) { INT i, j, k, n, *pc, *pc_offp; FLOAT *pd, *pd_offp, *vx, *vx0, *vb; FLOAT sum, omega = _p->smooth_damp;; VEC *x0; #if USE_MPI FLOAT *offp_data = NULL; #endif /* USE_MPI */ MagicCheck(VEC, x); MagicCheck(VEC, b); assert(x == NULL || x->nvec == 1); assert(b == NULL || b->nvec == 1); if (x != NULL && !x->assembled) phgVecAssemble(x); if (b != NULL && !b->assembled) phgVecAssemble(b); x0 = phgMapCreateVec(x->map, 1); assert(A->type != PHG_DESTROYED); if (!A->assembled) phgMatAssemble(A); assert(A->type != PHG_MATRIX_FREE); phgMatPack(A); #if USE_MPI if (A->cmap->nprocs > 1) { offp_data = phgAlloc(A->cinfo->rsize * sizeof(*offp_data)); } #endif /* USE_MPI */ if (A->cmap->nlocal != A->rmap->nlocal || A->cmap->nlocal != x->map->nlocal || A->cmap->nlocal != b->map->nlocal) phgError(1, "%s:%d: inconsistent matrix-vector.", __FILE__, __LINE__); #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ /* iteration */ for (k = 0; k < nsmooth; k++) { phgVecCopy(x, &x0); /* multiply with local data */ vx = x->data; vx0 = x0->data; vb = b->data; pc = A->packed_cols; pd = A->packed_data; if (A->cmap->nprocs > 1) { pc_offp = A->packed_cols + A->rmap->nlocal + A->nnz_d; pd_offp = A->packed_data + A->nnz_d; } else { pc_offp = NULL; pd_offp = NULL; } for (i = 0; i < A->rmap->nlocal; i++) { INT jcol; FLOAT aa = 0., dx; /* x_i = (b_i - \sum_{j ~= i} a_ij * x_j) / a_ii */ sum = vb[i]; /* local data */ if ((n = *(pc++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc++); if (jcol != i) { sum -= *(pd++) * vx0[jcol]; } else { aa = *(pd++); assert(fabs(aa) > 1e-14); } } } /* remote data */ if (pc_offp != NULL && (n = *(pc_offp++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc_offp++); sum -= *(pd_offp++) * offp_data[jcol]; } } dx = sum / aa - vx[i]; vx[i] += omega * dx; } #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ } phgVecDestroy(&x0); #if USE_MPI phgFree(offp_data); #endif /* USE_MPI */ return; }
/* * Gauss-Sidel smoother for vector: * * As GS: * 1. exchange off proc data * 2. smooth local dof * * Assumption: * 1. unknow dof is vector of dim 3 * 2. Matrix block for vector is * | a1 -b 0 | * | b a2 0 | * | 0 0 a3 | * * */ void mg_GaussSidel_vec(MAT *A, VEC *x, VEC *b, int nsmooth, void *ctx) { INT i, j, k, l, n, *pc, *pc0, *pc_offp, nlocal; FLOAT *pd, *pd0, *pd_offp, *vx, *vb; size_t *ps; FLOAT sum[Dim], omega = _p->smooth_damp; #if USE_MPI FLOAT *offp_data = NULL; #endif /* USE_MPI */ MagicCheck(VEC, x); MagicCheck(VEC, b); assert(x == NULL || x->nvec == 1); assert(b == NULL || b->nvec == 1); if (x != NULL && !x->assembled) phgVecAssemble(x); if (b != NULL && !b->assembled) phgVecAssemble(b); assert(A->type != PHG_DESTROYED); if (!A->assembled) phgMatAssemble(A); assert(A->type != PHG_MATRIX_FREE); phgMatPack(A); #if USE_MPI if (A->cmap->nprocs > 1) { offp_data = phgAlloc(A->cinfo->rsize * sizeof(*offp_data)); } #endif /* USE_MPI */ if (A->cmap->nlocal != A->rmap->nlocal || A->cmap->nlocal != x->map->nlocal || A->cmap->nlocal != b->map->nlocal) phgError(1, "%s:%d: inconsistent matrix-vector.", __FILE__, __LINE__); if (A->cmap->nlocal % Dim != 0) phgError(1, "%s: assume vector dof of dim 3!\n", __FUNCTION__); #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ /* iteration */ for (l = 0; l < nsmooth; l++) { INT i_start, i_add; /* multiply with local data */ vx = x->data; vb = b->data; pc0 = A->packed_cols; pd0 = A->packed_data; ps = A->packed_ind; nlocal = A->rmap->nlocal; /* * lexicographic order: low to high * Note: low->high and high->low alternatively does not help. * */ if (TRUE || l % 2 == 0) { i_start = 0; i_add = Dim; } else { i_start = nlocal - Dim; i_add = -Dim; } //for (i = i_start; i < A->rmap->nlocal && i >= 0; i += i_add) { for (i = 0; i < nlocal ; i += Dim) { INT jcol; FLOAT aa[Dim][Dim], det, dx[Dim]; memset(aa, 0, sizeof(aa)); sum[0] = vb[i ]; sum[1] = vb[i+1]; sum[2] = vb[i+2]; /* local data */ pc = pc0 + PACK_COL(ps, i); pd = pd0 + PACK_DAT(ps, i); for (k = 0; k < Dim; k++) { if ((n = *(pc++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc++); if (jcol < i || jcol > i+2) { sum[k] -= *(pd++) * vx[jcol]; } else { /* offD, jcol = i,i+1,i+2 */ aa[k][jcol - i] = *(pd++); } } } } /* remote data */ if (A->cmap->nprocs > 1) { pc_offp = pc0 + PACK_COL_OFFP(ps, i, nlocal); pd_offp = pd0 + PACK_DAT_OFFP(ps, i, nlocal); for (k = 0; k < Dim; k++) { if ((n = *(pc_offp++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc_offp++); sum[k] -= *(pd_offp++) * offp_data[jcol]; } } } } /* solve */ det = (aa[0][0] * aa[1][1] - aa[0][1] * aa[1][0]); assert(Fabs(det) > 1e-12); det = 1. / det; dx[0] = (aa[1][1] * sum[0] - aa[0][1] * sum[1]) * det - vx[i ]; dx[1] = (aa[0][0] * sum[1] - aa[1][0] * sum[1]) * det - vx[i+1]; dx[2] = (1./aa[2][2] * sum[2]) - vx[i+2]; vx[i ] += omega * dx[0]; vx[i+1] += omega * dx[1]; vx[i+2] += omega * dx[2]; } #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ } #if USE_MPI phgFree(offp_data); #endif /* USE_MPI */ return; }
/* * Gauss-Sidel smoother2: * * 1. interior dof (!REMOTE) is smoothed, and then offp data is updated; * 2. proc boundary dof (REMOTE) is smoothed, and then offp data is updated. * * Note: need types_vec. * */ void mg_GaussSidel2(MAT *A, VEC *x, VEC *b, int nsmooth, void *ctx) { MG_LEVEL *ml = (MG_LEVEL *)ctx; INT i, j, k, n, *pc, *pc_offp; FLOAT *pd, *pd_offp, *vx, *vb; FLOAT sum, omega = _p->smooth_damp;; #if USE_MPI FLOAT *offp_data = NULL; #endif /* USE_MPI */ MagicCheck(VEC, x); MagicCheck(VEC, b); assert(x == NULL || x->nvec == 1); assert(b == NULL || b->nvec == 1); if (x != NULL && !x->assembled) phgVecAssemble(x); if (b != NULL && !b->assembled) phgVecAssemble(b); assert(A->type != PHG_DESTROYED); if (!A->assembled) phgMatAssemble(A); assert(A->type != PHG_MATRIX_FREE); phgMatPack(A); #if USE_MPI if (A->cmap->nprocs > 1) { offp_data = phgAlloc(A->cinfo->rsize * sizeof(*offp_data)); } #endif /* USE_MPI */ if (A->cmap->nlocal != A->rmap->nlocal || A->cmap->nlocal != x->map->nlocal || A->cmap->nlocal != b->map->nlocal) phgError(1, "%s:%d: inconsistent matrix-vector.", __FILE__, __LINE__); #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ /* iteration */ for (k = 0; k < nsmooth; k++) { /* multiply with local data */ vx = x->data; vb = b->data; /* First, interior dof (!REMOTE) is smoothed, and then offp data is updated */ pc = A->packed_cols; pd = A->packed_data; if (A->cmap->nprocs > 1) { pc_offp = A->packed_cols + A->rmap->nlocal + A->nnz_d; pd_offp = A->packed_data + A->nnz_d; } else { pc_offp = NULL; pd_offp = NULL; } for (i = 0; i < A->rmap->nlocal; i++) { INT jcol; FLOAT aa = 0., dx; if (ml->types_vec[i] & REMOTE) { if ((n = *(pc++)) != 0) { pc += n; pd += n; } if (pc_offp != NULL && (n = *(pc_offp++)) != 0) { pc_offp += n; pd_offp += n; } continue; } sum = vb[i]; /* local data */ if ((n = *(pc++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc++); if (jcol != i) { sum -= *(pd++) * vx[jcol]; } else { aa = *(pd++); assert(fabs(aa) > 1e-14); } } } /* remote data */ if (pc_offp != NULL && (n = *(pc_offp++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc_offp++); sum -= *(pd_offp++) * offp_data[jcol]; } } dx = sum / aa - vx[i]; vx[i] += omega * dx; } #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ /* Second, proc boundary dof (!REMOTE) is smoothed, and then offp data is updated */ pc = A->packed_cols; pd = A->packed_data; if (A->cmap->nprocs > 1) { pc_offp = A->packed_cols + A->rmap->nlocal + A->nnz_d; pd_offp = A->packed_data + A->nnz_d; } else { pc_offp = NULL; pd_offp = NULL; } for (i = 0; i < A->rmap->nlocal; i++) { INT jcol; FLOAT aa = 0., dx; if (!(ml->types_vec[i] & REMOTE)) { if ((n = *(pc++)) != 0) { pc += n; pd += n; } if (pc_offp != NULL && (n = *(pc_offp++)) != 0) { pc_offp += n; pd_offp += n; } continue; } sum = vb[i]; /* local data */ if ((n = *(pc++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc++); if (jcol != i) { sum -= *(pd++) * vx[jcol]; } else { aa = *(pd++); assert(fabs(aa) > 1e-14); } } } /* remote data */ if (pc_offp != NULL && (n = *(pc_offp++)) != 0) { for (j = 0; j < n; j++) { jcol = *(pc_offp++); sum -= *(pd_offp++) * offp_data[jcol]; } } dx = sum / aa - vx[i]; vx[i] += omega * dx; } #if USE_MPI if (A->cmap->nprocs > 1) { phgMapScatterBegin(A->cinfo, x->nvec, x->data, offp_data); phgMapScatterEnd(A->cinfo, x->nvec, x->data, offp_data); } #endif /* USE_MPI */ } #if USE_MPI phgFree(offp_data); #endif /* USE_MPI */ return; }
/**************************************************************** * Build RHS which is the residual of the nonlinear system. ***************************************************************/ void phgNSBuildSolverURHS(NSSolver *ns) { GRID *g = ns->g; SIMPLEX *e; SOLVER *solver_u = ns->solver_u; int i, k, l, q, s; FLOAT *dt = ns->dt; BOOLEAN tstep_minus = (ns->u[-1] != NULL); VEC *vec_rhs = phgMapCreateVec(solver_u->rhs->map, 1); FLOAT Theta = _nsp->Theta, nu = _nsp->nu, Thet1; int viscosity_type = ns->viscosity_type; SURF_BAS *surf_bas = ns->surf_bas; DOF *surf_dof = surf_bas->dof; BOOLEAN *rotated = surf_bas->rotated; const FLOAT *Trans = DofData(surf_dof); #if STEADY_STATE assert(fabs(Theta - 1) < 1e-12); Thet1 = 0; Unused(Thet1); Unused(dt); #else Thet1 = 1 - Theta; Unused(dt); #endif /* STEADY_STATE */ phgPrintf(" DB_mask: ["); for (k = 0; k < Dim; k++) phgPrintf("%d ", ns->u[1]->DB_masks[k]); phgPrintf("] "); nu_max = -1e10; nu_min = +1e10; phgVecDisassemble(vec_rhs); ForAllElements(g, e) { int M = ns->u[1]->type->nbas; /* num of bases of Velocity */ int N = ns->p[1]->type->nbas; /* num of bases of Pressure */ int order = DofTypeOrder(ns->u[1], e) * 3 - 1; /* Note: * quad order is really high here, * highest order term (u \nabla u, phi) */ FLOAT bufu[M], bufp[N], rhsu[M][Dim], rhsp[N]; INT Iu[M][Dim], Ip[N]; QUAD *quad; FLOAT vol, area, det; const FLOAT *w, *p, *normal, **vu, *vu_queue[3], *vf[2], *gu[2], *vp[2], *vw, *vT; FLOAT *vf_cache[2]; vu = vu_queue + 1; quad = phgQuadGetQuad3D(order); vu[0] = phgQuadGetDofValues(e, ns->u[0], quad); /* u^{n} */ vp[0] = phgQuadGetDofValues(e, ns->p[0], quad); /* p^{n} */ gu[0] = phgQuadGetDofValues(e, ns->gradu[0], quad); /* grad u^{n} */ vw = phgQuadGetDofValues(e, ns->wind, quad); /* wind */ vT = phgQuadGetDofValues(e, ns->T[1], quad); /* T^{n} */ if (tstep_minus) { vu[-1] = phgQuadGetDofValues(e, ns->u[-1], quad); /* u^{n-1} */ } else { vu[-1] = vu[0]; } #if STEADY_STATE || TIME_DEP_NON vu[1] = phgQuadGetDofValues(e, ns->u[1], quad); /* u^{n+1} */ gu[1] = phgQuadGetDofValues(e, ns->gradu[1], quad); /* grad u^{n} */ vp[1] = phgQuadGetDofValues(e, ns->p[1], quad); /* p^{n+1} */ #else TIME_DEP_LINEAR_ENTRY; /* Unavailable */ #endif /* STEADY_STATE || TIME_DEP_NON */ Unused(l); Unused(vf); Unused(vf_cache); if (!_nsp->no_extern_source) { /* cache f values */ for (l = 0; l < 2; l++) { const FLOAT *cache; size_t cache_size; setFuncTime(ns->time[l]); /* set static time in ins-test.c */ /* cache f */ cache_size = Dim * quad->npoints * sizeof(FLOAT); cache = phgQuadGetFuncValues(g, e, Dim, func_f, quad); vf[l] = vf_cache[l] = phgAlloc(cache_size); memcpy(vf_cache[l], cache, cache_size); phgQuadGetFuncValues(NULL, NULL, 0, NULL, NULL); /* clear cache */ } } /* Global Matrix */ Bzero(rhsu); Bzero(rhsp); p = quad->points; w = quad->weights; for (q = 0; q < quad->npoints; q++) { phgGeomGetCurvedJacobianAtLambda(g, e, p, &det); vol = fabs(det / 6.); /* rhs u */ for (i = 0; i < M; i++) { /* interior node or Neumann */ const FLOAT *gi_u = phgQuadGetBasisValues(e, ns->u[1], i, quad) + q; /* phi_i */ const FLOAT *ggi_u = phgQuadGetBasisCurvedGradient(e, ns->u[1], i, quad, q); /* grad phi_i */ for (k = 0; k < Dim; k++) { #if ICE_BENCH_TEST nu = get_effective_viscosity(gu[1], 0, 0, viscosity_type); FLOAT eu[DDim]; MAT3_SYM(gu[1], eu); rhsu[i][k] += vol*(*w) * EQU_SCALING * (- nu * INNER_PRODUCT(eu+k*Dim, ggi_u) + (*vp[1]) * *(ggi_u+k) * LEN_SCALING * PRES_SCALING ); /* left */ if (k == Z_DIR) { const FLOAT rho = RHO_ICE; const FLOAT grav = GRAVITY; const FLOAT a = SEC_PER_YEAR; const FLOAT f = rho*grav * EQU_SCALING * LEN_SCALING2; Unused(a); rhsu[i][k] += vol*(*w) * (-f * (*gi_u) ); /* right */ } #elif ESIMINT_TEST || \ HEINO_TEST || \ TEST_CASE == ICE_GREEN_LAND nu = get_effective_viscosity(gu[1], *vT, 0, viscosity_type); FLOAT eu[DDim]; MAT3_SYM(gu[1], eu); rhsu[i][k] += vol*(*w) * EQU_SCALING * (- nu * INNER_PRODUCT(eu+k*Dim, ggi_u) + (*vp[1]) * *(ggi_u+k) * LEN_SCALING * PRES_SCALING ); /* left */ if (k == Z_DIR) { const FLOAT rho = RHO_ICE; const FLOAT grav = GRAVITY; const FLOAT a = SEC_PER_YEAR; const FLOAT f = rho*grav * EQU_SCALING * LEN_SCALING2; Unused(a); rhsu[i][k] += vol*(*w) * (-f * (*gi_u) ); /* right */ } #elif STEADY_STATE rhsu[i][k] += vol*(*w) * (- nu * INNER_PRODUCT(gu[1]+k*Dim, ggi_u) + (*vp[1]) * *(ggi_u+k) ); /* left */ if (!_nsp->no_extern_source) rhsu[i][k] += vol*(*w) * (*(vf[1]+k) * (*gi_u) ); /* right */ #elif TIME_DEP_NON rhsu[i][k] -= vol*(*w) * ((vu[1][k] - vu[0][k]) * (*gi_u) / dt[0] + Theta * (nu * INNER_PRODUCT(gu[1]+k*Dim, ggi_u) ) - (*vp[1]) * *(ggi_u+k) + Thet1 * (nu * INNER_PRODUCT(gu[0]+k*Dim, ggi_u) ) ); /* left */ if (!_nsp->no_extern_source) rhsu[i][k] += vol*(*w) * (Theta * *(vf[1]+k) * (*gi_u) + Thet1 * *(vf[0]+k) * (*gi_u) ); /* right */ #else TIME_DEP_LINEAR_ENTRY; /* Unavailable */ #endif /* STEADY_STATE */ } } /* rhs p */ for (i = 0; i < N; i++) { const FLOAT *gi_p = phgQuadGetBasisValues(e, ns->p[1], i, quad) + q; /* psi_i */ FLOAT divu1 = gu[1][0] + gu[1][4] + gu[1][8]; //FLOAT divu0 = gu[0][0] + gu[0][4] + gu[0][8]; rhsp[i] += vol*(*w) * (divu1 * (*gi_p) ); } if (tstep_minus) vu[-1] += Dim; #if STEADY_STATE || TIME_DEP_NON vu[1] += Dim; gu[1] += Dim*Dim; vp[1]++; #else TIME_DEP_LINEAR; /* Unavailable */ #endif /* STEADY_STATE || TIME_DEP_NON */ vu[0] += Dim; gu[0] += Dim * Dim; vp[0]++; vw += Dim; if (!_nsp->no_extern_source) { vf[0] += Dim; vf[1] += Dim; } vT++; w++; p += Dim + 1; } if (!_nsp->no_extern_source) { phgFree(vf_cache[0]); phgFree(vf_cache[1]); } normal = NULL; Unused(normal); area = 0; Unused(area); if (!_nsp->enclosed_flow) { /* slip boundary */ for (s = 0; s < NFace; s++) { if (e->bound_type[s] & INFLOW) { int v0, v1, v2; int nbas_face = NbasFace(ns->u[1]); SHORT bases[nbas_face]; FLOAT lambda[Dim + 1], x,y,z, beta; order = DofTypeOrder(ns->u[1], e) * 3 - 1; phgDofGetBasesOnFace(ns->u[1], e, s, bases); v0 = GetFaceVertex(s, 0); v1 = GetFaceVertex(s, 1); v2 = GetFaceVertex(s, 2); lambda[s] = 0.; area = phgGeomGetFaceArea(g, e, s); normal = phgGeomGetFaceOutNormal(g, e, s); quad = phgQuadGetQuad2D(order); p = quad->points; w = quad->weights; for (q = 0; q < quad->npoints; q++) { FLOAT vu[Dim]; lambda[v0] = *(p++); lambda[v1] = *(p++); lambda[v2] = *(p++); phgGeomLambda2XYZ(g, e, lambda, &x, &y, &z); func_beta(x, y, z, &beta); phgDofEval(ns->u[1], e, lambda, vu); for (i = 0; i < nbas_face; i++) { int ii = bases[i]; FLOAT gi_u = *ns->u[1]->type->BasFuncs(ns->u[1], e, ii, ii + 1, lambda); for (k = 0; k < Dim; k++) { #if STEADY_STATE rhus[ii][k] += 0.; #elif TIME_DEP_NON # if USE_SLIDING_BC abort(); rhsu[ii][k] += SIGN_FRICTION * area*(*w) * beta * vu[k] * (gi_u) * EQU_SCALING * LEN_SCALING; # else Unused(gi_u); # endif #else TIME_DEP_LINEAR_ENTRY; /* Unavailable */ #endif /* STEADY_STATE */ } } /* end of bas_i */ w++; } /* end of quad point */ } /* end of face outflow */ } /* end of all outflow face in element */ } /* end out flow boundary */ #if USE_SLIDING_BC /* Rotate bases */ for (i = 0; i < M; i++) { INT id = phgDofMapE2D(surf_dof, e, i * (Dim*Dim)) / (Dim*Dim); if (!rotated[id]) continue; const FLOAT *trans = Trans + id*(Dim*Dim); trans_left(&rhsu[i][0], 1, 1, trans); } #else Unused(Trans); Unused(rotated); #endif /* Map: Element -> system */ for (i = 0; i < M; i++) for (k = 0; k < Dim; k++) Iu[i][k] = phgMapE2L(solver_u->rhs->map, 0, e, i * Dim + k); for (i = 0; i < N; i++) Ip[i] = phgMapE2L(solver_u->rhs->map, 1, e, i); /* set velocity dirichlet bdry */ FLOAT tmp[Dim]; for (i = 0; i < M; i++) for (k = 0; k < Dim; k++) if (phgDofDirichletBC_(ns->u[1], e, i*Dim+k, NULL, bufu, tmp, DOF_PROJ_NONE)) { rhsu[i][k] = 0.; } #if STEADY_STATE || TIME_DEP_NON /* set pressure dirichlet bdry for pinned point */ for (i = 0; i < N; i++) if (phgDofDirichletBC(ns->p[1], e, i, NULL, bufp, &rhsp[i], DOF_PROJ_NONE)) { if (!_nsp->enclosed_flow) phgError(1, "No dirichlet bc for Unenclosed flow!\n"); if (_nsp->pin_node) { # if PIN_AT_ROOT if (g->rank != 0) phgError(1, "Pinned node only on rank 0!\n"); if (g, e->verts[i] != ns->pinned_node_id) phgError(1, "Build rhs: pinned node e:%d, bas:%d, [%d] and [%d] " "doesn't coincide when build RHS!\n", e->index, i, e->verts[i], ns->pinned_node_id); # else if (GlobalVertex(g, e->verts[i]) != ns->pinned_node_id) phgError(1, "Build rhs: pinned node e:%d, bas:%d, [%d] and [%d] " "doesn't coincide when build RHS!\n", e->index, i, e->verts[i], ns->pinned_node_id); # endif /* PIN_AT_ROOT */ } } #else TIME_DEP_LINEAR; /* Unavailable */ #endif /* STEADY_STATE || TIME_DEP_NON */ /* Global res */ phgVecAddEntries(vec_rhs, 0, M * Dim, Iu[0], &rhsu[0][0]); phgVecAddEntries(vec_rhs, 0, N, Ip, rhsp); } /* end element */
DOF * phgDofCopy_(DOF *src, DOF **dest_ptr, DOF_TYPE *newtype, const char *name, const char *srcfile, int srcline) /* returns a new DOF whose type is 'newtype', and whose values are evaluated * using 'src' (a copy of src). */ { GRID *g = src->g; SIMPLEX *e; FLOAT w = 1.0, *basvalues = NULL, *a, *d, *buffer = NULL; int i, k, dim, dim_new, count = 0, nvalues, nd; INT n; DOF *wgts = NULL; DOF *dest = (dest_ptr == NULL ? NULL : *dest_ptr); BYTE *flags0, *flags; char *auto_name; DOF_TYPE *oldtype; BOOLEAN basflag = FALSE; MagicCheck(DOF, dest) if (dest != NULL && newtype != NULL && dest->type != newtype) { phgDofFree(&dest); dest = NULL; } dim = DofDim(src); /* the name of dest */ if ((auto_name = (void *)name) == NULL) { #if 0 auto_name = phgAlloc(strlen(src->name) + 8 + 1); sprintf(auto_name, "copy of %s", src->name); #else auto_name = strdup(src->name); #endif } if (dest == NULL) { if (newtype == NULL) newtype = src->type; dim_new = (newtype == NULL ? DofTypeDim(src) : newtype->dim); assert(dim % dim_new == 0); dest = phgDofNew_(g, newtype, newtype == NULL ? src->hp : NULL, dim / dim_new, auto_name, DofNoAction, srcfile, srcline); if (dest_ptr != NULL) *dest_ptr = dest; } else { assert(dim == DofDim(dest)); phgFree(dest->name); dest->name = strdup(auto_name); dest->srcfile = srcfile; dest->srcline = srcline; phgFree(dest->cache_func); dest->cache_func = NULL; newtype = dest->type; if (!SpecialDofType(newtype)) memset(dest->data, 0, DofGetDataCount(dest) * sizeof(*dest->data)); } if (auto_name != name) phgFree(auto_name); phgDofClearCache(NULL, dest, NULL, NULL, FALSE); dest->DB_mask = src->DB_mask; if (src->DB_masks != NULL) { if (dest->DB_masks == NULL) dest->DB_masks = phgAlloc(dest->dim * sizeof(*dest->DB_masks)); memcpy(dest->DB_masks, src->DB_masks, dest->dim * sizeof(*dest->DB_masks)); } dest->invariant = src->invariant; if (SpecialDofType(newtype)) { assert(newtype == src->type); dest->userfunc = src->userfunc; dest->userfunc_lambda = src->userfunc_lambda; if (newtype == DOF_CONSTANT) memcpy(dest->data, src->data, dim * sizeof(*dest->data)); return dest; } if ((newtype != NULL && newtype == src->type) || (newtype == NULL && src->hp == dest->hp)) { /* simply duplicate the data */ size_t size = DofGetDataCount(dest); if (size > 0) memcpy(dest->data, src->data, sizeof(FLOAT) * size); dest->userfunc = src->userfunc; dest->userfunc_lambda = src->userfunc_lambda; return dest; } if (src->type == DOF_ANALYTIC) { if (src->userfunc_lambda != NULL) phgDofSetDataByLambdaFunction(dest, src->userfunc_lambda); else phgDofSetDataByFunction(dest, src->userfunc); return dest; } if (newtype != NULL && newtype->BasFuncs == NULL) phgError(1, "phgDofCopy: basis funcs for DOF type \"%s\" undefined.\n", newtype->name); dest->userfunc = src->userfunc; dest->userfunc_lambda = src->userfunc_lambda; oldtype = src->type; if (oldtype == NULL) oldtype = src->hp->info->types[src->hp->info->min_order]; if (!SpecialDofType(oldtype) && newtype != NULL && newtype->points != NULL && !DofIsHP(src)) { basflag = TRUE; count = oldtype->nbas * oldtype->dim; basvalues = phgAlloc(newtype->nbas * count * sizeof(*basvalues)); if (oldtype->invariant == TRUE) get_bas_funcs(src, dest, src->g->roots, basvalues); } if (newtype == NULL) newtype = dest->hp->info->types[dest->hp->max_order]; flags0 = phgCalloc((newtype->np_vert > 0 ? g->nvert : 0) + (newtype->np_edge > 0 ? g->nedge : 0) + (newtype->np_face > 0 ? g->nface : 0), sizeof(*flags0)); if (!SpecialDofType(oldtype) && oldtype->continuity < 0) { static DOF_TYPE DOF_WGTS = {DofCache, "Weights", NULL, NULL, NULL, NULL, NULL, NULL, NULL, phgDofInitFuncPoint, NULL, NULL, NULL, FE_None, FALSE, FALSE, -1, 0, 0, -1, 1, 0, 0, 0, 0 }; DOF_WGTS.np_vert = (newtype->np_vert > 0) ? 1 : 0; DOF_WGTS.np_edge = (newtype->np_edge > 0) ? 1 : 0; DOF_WGTS.np_face = (newtype->np_face > 0) ? 1 : 0; DOF_WGTS.nbas = DOF_WGTS.np_vert * NVert + DOF_WGTS.np_edge * NEdge + DOF_WGTS.np_face * NFace; if (DOF_WGTS.nbas > 0) { /* Other cases will be implemented later when really needed */ wgts = phgDofNew(g, &DOF_WGTS, 1, "weights", DofNoAction); phgDofSetDataByValue(wgts, 0.0); phgDofSetDataByValue(dest, 0.0); /* allocate buffer for storing weighted vertex/edge/face data */ if ((n = DofGetDataCount(dest) - DofGetElementDataCount(dest)) > 0) buffer = phgCalloc(n, sizeof(*buffer)); } } nvalues = dest->dim; cache_dof = src; bas_count = count; ForAllElements(g, e) { if (wgts != NULL) { #if 0 /* use element volume as weight */ w = phgGeomGetVolume(g, e); #else /* use 1 as weight */ w = 1.0; #endif } if (basflag && oldtype->invariant == FALSE) get_bas_funcs(src, dest, e, basvalues); bas = basvalues; flags = flags0; if (DofIsHP(dest)) newtype = dest->hp->info->types[dest->hp->elem_order[e->index]]; if (newtype->np_vert > 0) { nd = nvalues * newtype->np_vert; for (k = 0; k < NVert; k++) { if (flags[n = e->verts[k]] && wgts == NULL) { /* note: count==0 and bas==NULL for variable order DOF */ bas += count * newtype->np_vert; continue; } flags[n] = TRUE; a = DofVertexData(dest, n); newtype->InitFunc(dest, e, VERTEX, k, NULL, func, NULL, a, NULL); if (wgts != NULL) { d = buffer + (a - DofData(dest)); for (i = 0; i < nd; i++) *(d++) += *(a++) * w; *DofVertexData(wgts, n) += w; } } flags += g->nvert; } if (newtype->np_edge > 0) { nd = nvalues * newtype->np_edge; for (k = 0; k < NEdge; k++) { if (flags[n = e->edges[k]] && wgts == NULL) { /* note: count==0 and bas==NULL for variable order DOF */ bas += count * newtype->np_edge; continue; } flags[n] = TRUE; a = DofEdgeData(dest, n); newtype->InitFunc(dest, e, EDGE, k, NULL, func, NULL, a, NULL); if (wgts != NULL) { d = buffer + (a - DofData(dest)); if (DofIsHP(dest)) nd = dest->dim * (dest->hp->edge_index[n + 1] - dest->hp->edge_index[n]); for (i = 0; i < nd; i++) *(d++) += *(a++) * w; *DofEdgeData(wgts, n) += w; } } flags += g->nedge; } if (newtype->np_face > 0) { nd = nvalues * newtype->np_face; for (k = 0; k < NFace; k++) { if (flags[n = e->faces[k]] && wgts == NULL) { /* note: count==0 and bas==NULL for variable order DOF */ bas += count * newtype->np_face; continue; } flags[n] = TRUE; a = DofFaceData(dest, n); newtype->InitFunc(dest, e, FACE, k, NULL, func, NULL, a, NULL); if (wgts != NULL) { d = buffer + (a - DofData(dest)); if (DofIsHP(dest)) nd = dest->dim * (dest->hp->face_index[n + 1] - dest->hp->face_index[n]); for (i = 0; i < nd; i++) *(d++) += *(a++) * w; *DofFaceData(wgts, n) += w; } } } if (newtype->np_elem > 0) { a = DofElementData(dest, e->index); newtype->InitFunc(dest, e, ELEMENT, 0, NULL, func, NULL, a, NULL); } } phgFree(basvalues); phgFree(flags0); if (wgts == NULL) return dest; if ((n = DofGetDataCount(dest) - DofGetElementDataCount(dest)) > 0) { memcpy(DofData(dest), buffer, n * sizeof(*buffer)); phgFree(buffer); } if (DofIsHP(dest)) newtype = dest->hp->info->types[dest->hp->max_order]; a = DofData(dest); d = DofData(wgts); #if USE_MPI /* FIXME: directly interacting with the vector assembly code in solver.c is more efficient */ if (g->nprocs > 1) { SOLVER *solver = phgSolverCreate(SOLVER_BUILTIN, dest, NULL); INT K = 0, I; int j; if (newtype->np_vert > 0) { nd = dest->count_vert; for (n = 0; n < g->nvert; n++, d++) { if (g->types_vert[n] == UNREFERENCED) { K += nd; a += nd; continue; } for (j = 0; j < nd; j++, K++, a++) { I = phgSolverMapD2L(solver, 0, K); assert(I >= 0 && I < solver->mat->rmap->localsize); phgSolverAddRHSEntry(solver, I, *a); phgSolverAddMatrixEntry(solver, I, I, *d); } } } if (newtype->np_edge > 0) { nd = nvalues * newtype->np_edge; for (n = 0; n < g->nedge; n++, d++) { if (DofIsHP(dest)) nd = dest->dim * (dest->hp->edge_index[n + 1] - dest->hp->edge_index[n]); if (g->types_edge[n] == UNREFERENCED) { K += nd; a += nd; continue; } for (j = 0; j < nd; j++, K++, a++) { I = phgSolverMapD2L(solver, 0, K); assert(I >= 0 && I < solver->mat->rmap->localsize); phgSolverAddRHSEntry(solver, I, *a); phgSolverAddMatrixEntry(solver, I, I, *d); } } } if (newtype->np_face > 0) { nd = nvalues * newtype->np_face; for (n = 0; n < g->nface; n++, d++) { if (DofIsHP(dest)) nd = dest->dim * (dest->hp->face_index[n + 1] - dest->hp->face_index[n]); if (g->types_face[n] == UNREFERENCED) { K += nd; a += nd; continue; } for (j = 0; j < nd; j++, K++, a++) { I = phgSolverMapD2L(solver, 0, K); assert(I >= 0 && I < solver->mat->rmap->localsize); phgSolverAddRHSEntry(solver, I, *a); phgSolverAddMatrixEntry(solver, I, I, *d); } } } if (newtype->np_elem > 0) { nd = nvalues * newtype->np_elem; for (n = 0; n < g->nelem; n++) { if (DofIsHP(dest)) nd = dest->dim * (dest->hp->elem_index[n + 1] - dest->hp->elem_index[n]); if (g->types_elem[n] == UNREFERENCED) { K += nd; a += nd; continue; } for (j = 0; j < nd; j++, K++, a++) { I = phgSolverMapD2L(solver, 0, K); assert(I >= 0 && I < solver->mat->rmap->localsize); phgSolverAddRHSEntry(solver, I, *a); phgSolverAddMatrixEntry(solver, I, I, 1.0); } } } phgSolverSolve(solver, TRUE, dest, NULL); phgSolverDestroy(&solver); phgDofFree(&wgts); return dest; } #endif if (newtype->np_vert > 0) { k = nvalues * newtype->np_vert; for (n = 0; n < g->nvert; n++) { if ((w = *(d++)) == 0.0) { a += k; } else if (k == 1) { *(a++) /= w; } else { w = 1.0 / w; for (i = 0; i < k; i++) *(a++) *= w; } } } if (newtype->np_edge > 0) { k = nvalues * newtype->np_edge; for (n = 0; n < g->nedge; n++) { if (DofIsHP(dest)) k = dest->dim * (dest->hp->edge_index[n + 1] - dest->hp->edge_index[n]); if ((w = *(d++)) == 0.0) { a += k; } else if (k == 1) { *(a++) /= w; } else { w = 1.0 / w; for (i = 0; i < k; i++) *(a++) *= w; } } } if (newtype->np_face > 0) { k = nvalues * newtype->np_face; for (n = 0; n < g->nface; n++) { if (DofIsHP(dest)) k = dest->dim * (dest->hp->face_index[n + 1] - dest->hp->face_index[n]); if ((w = *(d++)) == 0.0) { a += k; } else if (k == 1) { *(a++) /= w; } else { w = 1.0 / w; for (i = 0; i < k; i++) *(a++) *= w; } } } phgDofFree(&wgts); return dest; }
int main(int argc, char *argv[]) { GRID *g; SIMPLEX *e; DOF **u, **p, **T, **gradu, *eu, *ep, *egradu, *ediv, *eT, *dH = NULL; FLOAT Time, *dt, res, non_du, non_dp, non_dT; INT tstep = 0, nelem; char mesh_file[100], hostname[256], data_file[100], data_u[100], data_p[100], data_T[100], data_Crd[100]; size_t mem, mem_peak; int verb; double tt[3], tt1[3]; /* ---------- NS ---------- */ NSSolver *ns = NULL; SURF_BAS *surf_bas = NULL; LAYERED_MESH *gL = NULL; //GEO_INFO *geo = NULL; /* MG_BLOCK_DOFS *bk = NULL; */ /* ================================================================================ * * Initialize Grid & parameters * * ================================================================================ */ /* Global (static) options */ Unused(verb); ns_params = phgParametersCreate(); phgInit(&argc, &argv); phgOptionsShowUsed(); g = phgNewGrid(-1); //phgSetPeriodicity(g, ns_params->periodicity); phgImportSetBdryMapFunc(my_bc_map); if (ns_params->resume) { phgResumeStage(g, &Time, &tstep, mesh_file, data_file); phgPrintf("================================\n\n"); phgPrintf("* RESUME from time:%E, tstep:%d\n", Time, tstep); phgPrintf("* mesh:%s\n", mesh_file); phgPrintf("* data:%s\n", data_file); phgPrintf("================================\n"); if (!phgImport(g, mesh_file, FALSE)) phgError(1, "can't read file \"%s\".\n", ns_params->fn); } else { phgPrintf("Using mesh: %s\n", ns_params->fn); if (!phgImport(g, ns_params->fn, FALSE)) phgError(1, "can't read file \"%s\".\n", ns_params->fn); } checkBdry(g); elapsed_time(g, FALSE, 0.); /* reset timer */ gethostname(hostname, sizeof(hostname)); printf("#%5d# runing PID %5d on %s \n", phgRank, getpid(), hostname); NsSolver_Options(); phgPrintf(" Pre-refine & repartition "); phgRefineAllElements(g, ns_params->pre_refines); /* Set Reynolds number */ Time = ns_params->time_start; /* default: 0 */ setFuncTime(Time); setFlowParameter(ns_params->Re, ns_params->nu, Time); /* ================================================================================ * * build ice grid * * ================================================================================ */ iceInit(g, &gL); phgPrintf("Geometry initialization done!\n"); phgExportVTK(g, "ice_domain.vtk", NULL, NULL); /* ================================================================================ * * Create INS solver * * ================================================================================ */ /* Note: pointers u, p, gradu, dt * DIRECTLY access private member of INS solver. */ phgPrintf(" Create INS solver"); tstep = 1; /* time step start at 1 */ setFuncTime(Time); /* in file ins-bc.c: static */ //data_sur_T = read_txt_data("./LAS_temp_for_mesh.txt"); ns = phgNSCreate(g, ns_params); //get_mask_bot(ns); ns->time[1] = Time; u = ns->u; p = ns->p; T = ns->T; gradu = ns->gradu; dH = ns->dH; dt = ns->dt; /* direct accses ns */ dt[0] = ns_params->dt0; ns->gL = gL; //ns->bk = bk = NULL; //init_line_block(T[1], gL); /* Use line block */ elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); /* Init height & depth */ if (gL != NULL){ get_height_depth(ns); } /* surf bases */ //surf_bas = ns->surf_bas; #if 0 DOF *beta = phgDofNew(g, DOF_P1, 1, "beta", func_beta); phgExportEnsight(g, "check", beta, NULL); phgFinalize(); exit(1); #endif /* ------------------------------------------------------------ * * Resume dof data. * * ------------------------------------------------------------ */ if (ns_params->resume) { FILE *fp = NULL; char fname[1000]; phgResumeStage(g, &Time, &tstep, mesh_file, data_file); /* resume coord */ { const FLOAT *v = DofData(ns->coord); int i, k; //sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", tstep - 1); sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", 2); assert(ns->coord->type == DOF_P1); load_dof_data3(g, ns->coord, data_Crd, mesh_file); for (i = 0; i < g->nvert; i++) for (k = 0; k < Dim; k++) g->verts[i][k] = *(v++); phgGeomInit_(g, TRUE); } #if 1 /* resmue u_{n-1} */ //sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep - 2); sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 1); DATA_FILE_SURFIX; sprintf(fname, "%s.p%03d", data_u, g->rank); if ((fp = fopen(fname, "r")) == NULL) { phgPrintf("* u_{%d} unavailable.\n", tstep - 2); } else { fclose(fp); phgDofCopy(u[1], &u[0], NULL, "u_{n}"); phgDofCopy(p[1], &p[0], NULL, "p_{n}"); phgDofCopy(T[1], &T[0], NULL, "T_{n}"); gradu[0] = NULL; load_dof_data3(g, u[0], data_u, mesh_file); load_dof_data3(g, p[0], data_p, mesh_file); load_dof_data3(g, T[0], data_T, mesh_file); phgPrintf(" Resume u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n", tstep - 2, DofGetDataCountGlobal(u[0]), phgDofNormL2(u[0]), tstep - 2, DofGetDataCountGlobal(p[0]), phgDofNormL2(p[0])); phgPrintf(" Resume T_{%5d}[%8d]:%24.12E\n", tstep - 2, DofGetDataCountGlobal(T[0]), phgDofNormL2(T[0])); phgDofGradient(u[0], &gradu[0], NULL, "gradu_{n}"); phgDofSetFunction(u[0], DofInterpolation); phgDofSetFunction(p[0], DofInterpolation); //phgDofSetBdryDataByFunction(u[0], func_u, SETFLOW); DOF_SCALE(u[0], "resume"); DOF_SCALE(p[0], "resume"); DOF_SCALE(T[0], "resume"); DOF_SCALE(gradu[0], "resume"); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); } /* resmue u_{n} */ //sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep - 1); sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 2); DATA_FILE_SURFIX; sprintf(fname, "%s.p%03d", data_u, g->rank); if ((fp = fopen(fname, "r")) == NULL) { phgError(1, "read Dof data %s failed!\n", data_file); } else { fclose(fp); load_dof_data3(g, u[1], data_u, mesh_file); load_dof_data3(g, p[1], data_p, mesh_file); load_dof_data3(g, T[1], data_T, mesh_file); phgPrintf(" Resume u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n", tstep - 1, DofGetDataCountGlobal(u[1]), phgDofNormL2(u[1]), tstep - 1, DofGetDataCountGlobal(p[1]), phgDofNormL2(p[1])); phgPrintf(" Resume T_{%5d}[%8d]:%24.12E\n", tstep - 1, DofGetDataCountGlobal(T[1]), phgDofNormL2(T[1])); phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}"); phgDofSetFunction(u[1], DofInterpolation); phgDofSetFunction(p[1], DofInterpolation); //phgDofSetBdryDataByFunction(u[1], func_u, SETFLOW); DOF_SCALE(u[1], "resume"); DOF_SCALE(p[1], "resume"); DOF_SCALE(T[1], "resume"); DOF_SCALE(gradu[1], "resume"); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); } #endif /* Re init height & depth */ if (gL != NULL) { get_height_depth(ns); build_layered_mesh_height(g, gL); check_height(g, gL); } /* reconstruct last time step */ //Time -= dt[0]; Time = tstep - 1; ns->time[1] = Time; ns->time[0] = Time; setFuncTime(Time); #if 0 phgExportEnsightT(g, OUTPUT_DIR "/ins_" NS_PROBLEM , tstep, tstep, u[1], p[1], NULL); phgFinalize(); return 0; #endif /* debug exit */ } /* end of resume */ /* Init temp field */ DOF_SCALE(ns->beta, "test"); if (ns_params->solve_temp && tstep == 1) { phgPrintf("Init temp field!\n"); phgNSTempInit(ns); } /* ================================================================================ * * * Main loop: * 1. Steady state: adaptive refinement. * 2. Time dependent: time advance. * * ================================================================================ */ while (TRUE) { FLOAT all_memory_usage = phgMemoryUsage(g, NULL)/(1024.0*1024.0); if (all_memory_usage > 10000) break; get_surf_bot_elev(ns); DOF *grad_surf_elev = phgDofGradient(ns->surf_elev_P1, NULL, NULL, "gradu_surf_elev"); ns->grad_surf_elev = grad_surf_elev; //phgPrintf("modify mask bot!!\n"); //modify_mask_bot(ns); //modify_mask_bot(ns); //phgDofSetDataByValue(u[1], 0.); //phgDofSetDataByValue(p[1], 0.); ns->surf_bas = get_surface_bases(g, DOF_P2); surf_bas = ns->surf_bas; static BOOLEAN initialized = FALSE; FLOAT time_end = ns_params->time_end; elapsed_time(g, FALSE, 0.); /* reset timer */ phgGetTime(tt); if (Fabs(time_end - Time) < 1e-12) { phgPrintf("\n=======\nTime reach end: %lf, exit.\n", Time); phgPrintf("Time End %f\n", time_end); break; } if (tstep > ns_params->max_tstep) { phgPrintf("\n=======\nTime step reach end: %d, exit.\n", tstep); break; } #if 0 /* use time t^{n+1} */ dt[-1] = dt[0]; if (Time + dt[0] > time_end) dt[0] = time_end - Time; Time += dt[0]; setFuncTime(Time); #endif phgPrintf("\n==========\ntime: %lf, step:%d\n", (double)Time, tstep); phgPrintf(" %d DOF (u:%d, p:%d), %d elements, %d submesh%s, load imbalance: %lg\n", DofGetDataCountGlobal(u[1]) + DofGetDataCountGlobal(p[1]), DofGetDataCountGlobal(u[1]), DofGetDataCountGlobal(p[1]), g->nleaf_global, g->nprocs, g->nprocs > 1 ? "es" : "", (double)g->lif); /* save mesh */ if (ns_params->record && tstep % ns_params->step_span == 0) { phgResumeLogUpdate(g, &Time, &tstep, ns_params->fn, NULL); } if (!initialized) { /* reset mem_peak */ phgMemoryUsageReset(); initialized = TRUE; } /* ------------------------------------------------------------ * * Time marching * * ------------------------------------------------------------ */ /* update variale, * call this routine after time update and/or grid change.*/ phgNSTimeAdvance(ns, Time, tstep); phgPrintf(" update solution"); elapsed_time(g, TRUE, 0.); if (ns_params->pin_node) phgNSPinNode(ns); /* -------------------------------------------------------------------------------- * * Step 3. * * Momentum Equations. * * -------------------------------------------------------------------------------- */ int mask_iter = 0, inverse_iter = 0; INT IF_CHANGE_MASK; #if 1 while (TRUE) { // iteration for ice shelf mask updating and inversion if ((inverse_iter % 2) == 0) ns->set_dirichlet_bc = 0; /* newton bc for the surface, i.e. normal simulation */ else ns->set_dirichlet_bc = 1; /* constrain the surface with obs. vel. for inversion */ phgPrintf("----------------------------\n"); phgPrintf("ice shelf mask iteration: %d\n", mask_iter); phgPrintf("----------------------------\n"); elapsed_time(g, FALSE, 0.); /* reset timer */ /* * non-linear iteration. * */ int max_nonstep = 0, newton_start = 0; assert(ns_params->utype == DOF_P2); /* For nonlinear iter */ int nonstep = 0; non_du = non_dp = non_dT = 1e+10; DOF *u_last = phgDofCopy(u[1], NULL, NULL, "u_last"); DOF *p_last = phgDofCopy(p[1], NULL, NULL, "p_last"); FLOAT non_res_last = 1.; LTYPE ltype_last = PICARD; /* First step, change max non step. */ if (tstep == 1) { if (ns_params->max_nonstep0 > 0) max_nonstep = ns_params->max_nonstep0; else max_nonstep = ns_params->max_nonstep; if (ns_params->newton_start0 > 0) newton_start = ns_params->newton_start0; else newton_start = ns_params->newton_start; phgPrintf(" * Set max nonstep to %d for first step.\n", max_nonstep); phgPrintf(" * Set Newton start to %d for first step.\n", newton_start); } else { max_nonstep = ns_params->max_nonstep; newton_start = ns_params->newton_start; } while (TRUE) { phgPrintf("\n ==================\n"); phgPrintf(" Non-linear interation step: %d\n", nonstep); /* Init const viscosity */ if (ns_params->start_const_vis && tstep == 0 && nonstep == 0 && mask_iter == 0) { phgPrintf("* vis: const\n"); ns->viscosity_type = VIS_CONST; } else { phgPrintf("* vis: strain\n"); ns->viscosity_type = VIS_STRAIN; } sayHello("Non linear solve begin"); phgNSInitSolverU(ns); if (inverse_iter < 1) get_viscosity(ns); /* initiate viscosity field at the beginning * afterwards use the updated viscosity field */ if (nonstep < newton_start) ns->ltype = PICARD; else ns->ltype = NEWTON; phgPrintf(" Build RHS: "); phgNSBuildSolverURHS(ns, 1, nonstep, Time); //phgNSBuildSolverURHS(ns); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); ns->non_res = res = phgVecNorm2(ns->solver_u->rhs, 0, NULL); phgPrintf(" nonlinear residual: %24.12E\n", res); /* Restore Picard if no improvement */ #if 1 if (ltype_last == NEWTON && res > non_res_last * .75) { phgPrintf(" !!! Newton step failed, use Picard to run again\n"); ns->ltype = PICARD; max_nonstep += 5; /* Add more Picard steps */ /* resotre dofs: * Fix me: temprature */ phgDofCopy(u_last, &u[1], NULL, "u_{n+1}"); phgDofCopy(p_last, &p[1], NULL, "p_{n+1}"); phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}"); phgNSBuildSolverURHS(ns, 1, nonstep, Time); ns->non_res = res = phgVecNorm2(ns->solver_u->rhs, 0, NULL); phgPrintf(" nonlinear residual: %24.12E\n", res); } #endif /* save non res */ non_res_last = res; ltype_last = ns->ltype; /* build matrices */ //if (ns_params->use_PCD) //phgNSInitPc(ns); phgPrintf(" Build matrices:\n"); phgNSBuildSolverUMat(ns, 1, nonstep, Time); //phgNSBuildSolverUMat(ns); phgPrintf(" done "); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); #if 0 if (ns_params->use_PCD) { phgPrintf(" Build Pc: \n"); phgNSBuildPc(ns); phgPrintf(" done "); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); } #endif //elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); /* * solve equation and update (u, p) * */ phgPrintf("solver tol: %E\n", ns->solver_u->rtol); phgSolverSolve(ns->solver_u, TRUE, ns->du, ns->dp, NULL); // elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); //get_water_pressure(ns); #if USE_NODAL_LOADS get_nodal_force(ns); phgMatDestroy(&ns->matF0); phgMatDestroy(&ns->matB0); phgMatDestroy(&ns->matBt0); phgMatDestroy(&ns->matC0); //phgSolverDestroy(&ns->solver_u0); ns->solver_u0 = NULL; #endif //get_contact(ns); #if USE_SLIDING_BC rotate_dof_bases(ns->du, surf_bas, FALSE); #endif phgPrintf(" solver_u: nits = %d, resid = %0.4lg ", ns->solver_u->nits, ns->solver_u->residual); //elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); /* save dofs */ phgDofCopy(u[1], &u_last, NULL, "u_last"); phgDofCopy(p[1], &p_last, NULL, "p_last"); phgExportVTK(g, "u_test.vtk", ns->u[1], NULL); /* nonlinear correction */ phgDofAXPY(1.0, ns->du, &u[1]); phgDofAXPY(1.0, ns->dp, &p[1]); assert(u[1]->type == ns_params->utype); assert(p[1]->type == ns_params->ptype); #if USE_SLIDING_BC //dof_set_normal_data(u[1], surf_bas); #else #endif /* non_du = phgDofNormL2(ns->du); */ /* non_dp = phgDofNormL2(ns->dp); */ non_du = phgDofNormInftyVec(ns->du); non_dp = phgDofNormInftyVec(ns->dp); phgPrintf(" \n du: %24.12E dp: %24.12E\n", non_du, non_dp); phgPrintf(" u: [%24.12E, %24.12E]\n", phgDofMinValVec(u[1]), phgDofMaxValVec(u[1])); phgPrintf(" p: [%24.12E, %24.12E]\n", phgDofMinValVec(p[1]), phgDofMaxValVec(p[1])); phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}"); //get_avg_gu(ns); //gradu[1] = ns->avg_gu; //elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); //if (ns_params->use_PCD) //phgNSDestroyPc(ns); // /* evolution of u */ //DOF_SCALE(u[1], "after solve"); //DOF_SCALE(p[1], "after solve"); #if 0 # warning check solution U,p sprintf(vtk_file, OUTPUT_DIR "non_%02d_u.vtk", nonstep); phgExportVTK(g, vtk_file, u[1], p[1], NULL); phgExportEnsightT(g, OUTPUT_DIR "ins_" NS_PROBLEM , nonstep, nonstep, u[1], p[1], T[1], ns->du, ns->dp, ns->dT, NULL); #endif #if 0 if (FALSE && nonstep % ns_params->step_span == 0) { phgPrintf(" Output solution to ensight "); phgExportEnsightT(g, OUTPUT_DIR "/ins_" NS_PROBLEM , nonstep, nonstep, u[1], p[1], T[1], NULL); /* ensight */ sprintf(vtk_file, OUTPUT_DIR "non_%02d_T.vtk", nonstep); phgExportVTK(g, vtk_file , u[1], p[1], T[1], ns->du, NULL); elapsed_time(g, TRUE, 0.); //ice_monitor(ns, nonstep); } #endif /* Linearized */ #if 0 if (!ns_params->non_linear && nonstep >= 0) { phgPrintf(" Linearized iteration converges.\n"); break; } #endif phgGetTime(tt1); phgPrintf(" time usage of current non step: %lfs\n", (double)(tt1[2] - tt[2])); nonstep++; /* * Nonliner iteration break, * converge for characteristic value. * Velocity: 100 m/a * Pressure: 1e8 Pa * * */ INT u_convergence; if (mask_iter < 1) u_convergence = check_u_convergence0(ns, u[1], p[1], u_last, p_last, ns_params->u_tol0, ns_params->u_tol0); else u_convergence = check_u_convergence(ns, u[1], p[1], u_last, p_last, ns_params->u_tol, ns_params->u_tol); //elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); const FLOAT U0 = 100; const FLOAT P0 = 1e8; if ( nonstep >= ns_params->min_nonstep && ns->viscosity_type != VIS_CONST && u_convergence || nonstep > ns_params->max_nonstep) { if (nonstep > ns_params->max_nonstep) { phgPrintf(" Non-linear iteration reach max step," " results may be inaccrate!\n"); #if 1 && USE_NODAL_LOADS /* get_nodal_force(ns); phgMatDestroy(&ns->matF0); phgMatDestroy(&ns->matB0); phgMatDestroy(&ns->matBt0); phgMatDestroy(&ns->matC0); //phgSolverDestroy(&ns->solver_u0); ns->solver_u0 = NULL; */ phgMatDestroy(&ns->matF); phgMatDestroy(&ns->matB); phgMatDestroy(&ns->matBt); phgMatDestroy(&ns->matC); phgSolverDestroy(&ns->solver_u); ns->solver_u = NULL; phgMapDestroy(&ns->Vmap); phgMapDestroy(&ns->Pmap); phgDofFree(&ns->u_shape); phgDofFree(&ns->p_shape); #endif break; } else { phgPrintf(" Non-linear iteration converges.\n"); #if 1 && USE_NODAL_LOADS /* get_nodal_force(ns); phgMatDestroy(&ns->matF0); phgMatDestroy(&ns->matB0); phgMatDestroy(&ns->matBt0); phgMatDestroy(&ns->matC0); //phgSolverDestroy(&ns->solver_u0); ns->solver_u0 = NULL; */ phgMatDestroy(&ns->matF); phgMatDestroy(&ns->matB); phgMatDestroy(&ns->matBt); phgMatDestroy(&ns->matC); phgSolverDestroy(&ns->solver_u); ns->solver_u = NULL; phgMapDestroy(&ns->Vmap); phgMapDestroy(&ns->Pmap); phgDofFree(&ns->u_shape); phgDofFree(&ns->p_shape); #endif break; } } phgNSDestroySolverU(ns); } /* solve */ phgDofFree(&u_last); phgDofFree(&p_last); //phgPrintf("Save Dofs\n"); //save_dof_data3(g, u[1], OUTPUT_DIR"u.dat"); //save_dof_data3(g, p[1], OUTPUT_DIR"p.dat"); #elif 0 /* Set velocity, for debugging */ ns->viscosity_type = VIS_STRAIN; phgDofSetDataByFunction(u[1], func_u0); //phgDofSetDataByFunction(p[1], func_p0); phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}"); #else phgPrintf("Load Dofs\n"); load_dof_data3(g, u[1], OUTPUT_DIR"u.dat", NULL); load_dof_data3(g, p[1], OUTPUT_DIR"p.dat", NULL); #endif /* Project to continuous gradu */ #if 1 phgPrintf("\n ======================= \n"); phgPrintf("project velocity gradient"); phgPrintf("\n======================== \n"); proj_gradu(ns, ns->gradu[1]); #endif //DOF_SCALE(gradu[1], "grad u"); //DOF_SCALE(ns->Gradu, "Grad u"); phgPrintf("\n----------------------------\n"); phgPrintf("check ice shelf mask status"); phgPrintf("\n----------------------------\n"); get_mask_bot(ns); if (!(ns_params->another_run_with_updated_mask)) { mask_iter = 1; phgPrintf("manually stops another run with updated mask!\n\n\n"); IF_CHANGE_MASK = if_update_shelf_mask(ns); IF_CHANGE_MASK = 0; get_mask_bot(ns); if (tstep % ns_params->step_span == 0) { phgPrintf("Save water and nodal forces to VTK \n"); DOF *water_P1 = phgDofCopy(ns->water_force, NULL, DOF_P1, NULL); DOF *nodal_P1 = phgDofCopy(ns->nodal_force, NULL, DOF_P1, NULL); sprintf(vtk_file, MASK_OUTPUT_DIR "MASK_%05d.vtk", tstep); phgExportVTK(g, vtk_file, water_P1,nodal_P1,ns->contact_force,ns->mask_bot,NULL); phgDofFree(&water_P1);phgDofFree(&nodal_P1); } } if (mask_iter < 1) { IF_CHANGE_MASK = if_update_shelf_mask(ns); } if ((ns_params->another_run_with_updated_mask)){ if (IF_CHANGE_MASK == 0) { phgPrintf("The lower surface mask remains unchanged. Stop the iteration of mask updating !\n"); if (tstep % ns_params->step_span == 0) { get_mask_bot(ns); phgPrintf("Save water and nodal forces to VTK \n"); DOF *water_P1 = phgDofCopy(ns->water_force, NULL, DOF_P1, NULL); DOF *nodal_P1 = phgDofCopy(ns->nodal_force, NULL, DOF_P1, NULL); sprintf(vtk_file, MASK_OUTPUT_DIR "MASK_%05d.vtk", tstep); phgExportVTK(g, vtk_file, water_P1,nodal_P1,ns->contact_force,ns->mask_bot,NULL); phgDofFree(&water_P1);phgDofFree(&nodal_P1); } phgDofFree(&ns->nodal_force); phgDofFree(&ns->water_force); phgDofFree(&ns->contact_force); break; } } get_strain_rate(ns); if (ns->set_dirichlet_bc) { //DOF *u_d = phgDofCopy(ns->u[1], NULL, DOF_P2, NULL); DOF *eu_d = phgDofCopy(ns->strain_rate, NULL, DOF_P1, NULL); ns->eu_d = eu_d; } else { //DOF *u_n = phgDofCopy(ns->u[1], NULL, DOF_P2, NULL); DOF *eu_n = phgDofCopy(ns->strain_rate, NULL, DOF_P1, NULL); ns->eu_n = eu_n; } DOF *visc_old = phgDofCopy(ns->viscosity, NULL, DOF_P1, NULL); if (inverse_iter > 0) update_viscosity_inversion(ns); FLOAT tol = 0.01; INT visc_convergence = check_visc_convergence(ns, visc_old, tol); if((mask_iter >= 1) && (inverse_iter > 0) && (visc_convergence == 1)) { phgPrintf("\n-------------------------\n"); phgPrintf("ice shelf mask updated \n"); phgPrintf("--------------------------\n"); break; } if (mask_iter < 1) if (tstep % ns_params->step_span == 0) { phgPrintf("Save water and nodal forces to VTK \n"); DOF *water_P1 = phgDofCopy(ns->water_force, NULL, DOF_P1, NULL); DOF *nodal_P1 = phgDofCopy(ns->nodal_force, NULL, DOF_P1, NULL); sprintf(vtk_file, MASK_OUTPUT_DIR "MASK_%05d.vtk", tstep); phgExportVTK(g, vtk_file, water_P1,nodal_P1,ns->contact_force,ns->mask_bot,NULL); phgDofFree(&water_P1);phgDofFree(&nodal_P1); } phgDofFree(&ns->nodal_force); phgDofFree(&ns->water_force); phgDofFree(&ns->contact_force); //phgDofFree(&ns->mask_bot); //phgDofFree(&ns->water_pressure); //phgDofFree(&ns->stress_nn); // phgPrintf("Free stress_nn !\n"); //phgDofFree(&ns->stress); //phgDofFree(&ns->water_pressure1); //phgDofFree(&ns->stress_nn1); //phgDofFree(&ns->stress1); //phgDofFree(&ns->avg_gu); mask_iter++; inverse_iter++; } proj_gradu(ns, ns->gradu[1]); get_stress(ns, ns->gradu[1], ns->p[1]); #if 0 if (1) { int i, k; DOF *Gu[DDim], *gu[DDim], *guDG0, *stress[DDim]; guDG0 = phgDofCopy(ns->gradu[1], NULL, DOF_P0, NULL); for (k = 0; k < DDim; k++) { FLOAT *vGu; INT n; char name[1000]; sprintf(name, "Gu%d", k); Gu[k] = phgDofNew(g, DOF_P1, 1, name, DofNoAction); vGu = ns->Gradu->data; /* DOF_P1 */ n = DofGetDataCount(Gu[k]); for (i = 0; i < n; i++) Gu[k]->data[i] = vGu[i * DDim + k]; sprintf(name, "stress%d", k); stress[k] = phgDofNew(g, DOF_P1, 1, name, DofNoAction); vGu = ns->stress->data; /* DOF_P1 */ n = DofGetDataCount(stress[k]); for (i = 0; i < n; i++) stress[k]->data[i] = vGu[i * DDim + k]; sprintf(name, "gu%d", k); gu[k] = phgDofNew(g, DOF_P0, 1, name, DofNoAction); vGu = guDG0->data; /* DOF_P0 */ n = DofGetDataCount(gu[k]); for (i = 0; i < n; i++) gu[k]->data[i] = vGu[i * DDim + k]; } phgExportVTK(g, "stress.vtk", stress[0], stress[1], stress[2], stress[3], stress[4], stress[5], stress[6], stress[7], stress[8], NULL); phgExportVTK(g, "Gu.vtk", Gu[0], Gu[1], Gu[2], Gu[3], Gu[4], Gu[5], Gu[6], Gu[7], Gu[8], gu[0], gu[1], gu[2], gu[3], gu[4], gu[5], gu[6], gu[7], gu[8], NULL); // phgFinalize(); } #endif /* -------------------------------------------------------------------------------- * * Step 4. * * Solve temperature. * * -------------------------------------------------------------------------------- */ #if 1 if (ns_params->solve_temp) { phgPrintf("\n ==================\n"); phgPrintf(" Temperature solve \n"); phgPrintf(" ==================\n\n"); phgPrintf(" T type: %s\n", T[1]->type->name); elapsed_time(g, FALSE, 0.); /* reset timer */ phgNSInitSolverT(ns); phgPrintf(" Build Mat: "); phgNSBuildSolverTMat(ns, FALSE); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); phgPrintf(" Build RHS: "); phgNSBuildSolverTRHS(ns, FALSE); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); phgNSSolverTBuildConstrain(ns); phgDofCopy(ns->T[1], &ns->dT, NULL, "dT"); phgNSSolverTSolve(ns, FALSE); elapsed_time(g, TRUE, phgPerfGetMflops(g, NULL, NULL)); phgNSDestroySolverT(ns); //find_melt_region(ns); DOF_SCALE(ns->T[1], "after solve"); phgDofAXPY(-1.0, ns->T[1], &ns->dT); non_dT = phgDofNormInftyVec(ns->dT); phgPrintf(" dT: %24.12E\n", non_dT); DOF *temp_diff = phgDofCopy(ns->T[1], NULL, NULL, "Td"); { FLOAT *vt = temp_diff->data; const FLOAT *vh = ns->depth_P2->data; INT i, n = DofGetDataCount(temp_diff); for (i = 0; i < n; i++, vh++, vt++) *vt = TEMP_WATER - BETA_MELT * (*vh) *LEN_SCALING - (*vt); } } else { phgPrintf("Temp not updated.\n"); non_dT = 0.; } #endif /* ------------------------------------------------------------ * * Error check * * ------------------------------------------------------------ */ #if 0 if (!ns_params->compute_error) { eu = ep = egradu = eT = NULL; ediv = phgDofDivergence(u[1], NULL, NULL, "err div u"); phgPrintf( " normL2(u, p) = (%20.12E, %20.12E)\n" " normH1(u) = (%20.12E)\n" " normDiv(u) = (%20.12E)\n" " normGadu = (%20.12E)\n", dofNormL2(u[1]), dofNormL2(p[1]), dofNormL2(gradu[1]), dofNormL2(ediv), dofNormL2(ns->gradu[1])); elapsed_time(g, TRUE, 0.); } else { /* ----Error check------- */ phgPrintf(" Errors: \n"); eu = phgDofCopy(u[1], NULL, ns_params->utype, "erru"); ep = phgDofCopy(p[1], NULL, ns_params->ptype, "errp"); eT = phgDofCopy(T[1], NULL, NULL, "errT"); egradu = phgDofCopy(gradu[1], NULL, NULL, "err grad u"); ediv = phgDofDivergence(u[1], NULL, NULL, "err div u"); adjust_time(- (1. - ns_params->Theta) * dt[0]); restore_time(); phgPrintf(" errL2(u, p) = (%20.12E, %20.12E)\n" " errH1(u) = (%20.12E)\n" " errDiv(u) = (%20.12E)\n", dofNormL2(eu), dofNormL2(ep), dofNormL2(egradu), dofNormL2(ediv)); elapsed_time(g, TRUE, 0.); phgDofFree(&egradu); dof_norm_L2(eT); } #endif getPecletNum(g, u[1], ns_params->nu, 6); mem = phgMemoryUsage(g, &mem_peak); phgPrintf(" Memory usage: current %0.4lgMB, peak %0.4lgMB\n", (double)mem / (1024.0 * 1024.0), (double)mem_peak / (1024.0 * 1024.0)); /* ------------------------------------------------------------ * * Move mesh * * ------------------------------------------------------------ */ FLOAT ice_volume_last = get_ice_volume(g); // DOF *dH_last = phgDofCopy(ns->dH, NULL, NULL, "dH_last"); if (ns_params->solve_height) { if (gL != NULL) { /* Unstructed layered mesh */ phgPrintf("Move mesh.\n"); get_surf_dH(ns); //DOF *dH_fem = phgDofCopy(ns->dH, NULL, NULL, NULL); phgExportVTK(g, "dH_fem1.vtk", ns->dH, NULL); get_smooth_surface_values(ns, ns->dH, 0); get_smooth_surface_values(ns, ns->dH, 1); phgExportVTK(g, "dH_fem.vtk", ns->dH, NULL); /* save_free_surface_elev(ns, 0); save_free_surface_elev(ns, 1); save_free_surface_velo(ns, 0, 0); save_free_surface_velo(ns, 1, 0); save_free_surface_velo(ns, 2, 0); save_free_surface_velo(ns, 0, 1); save_free_surface_velo(ns, 1, 1); save_free_surface_velo(ns, 2, 1); if (phgRank == 0) { system("python get_upper_ds.py"); system("python get_lower_ds.py"); } load_dH_from_file(ns, ns->dH, 0); load_dH_from_file(ns, ns->dH, 1); //DOF *dH_fdm = phgDofCopy(ns->dH, NULL, NULL, NULL); //phgDofAXPY(-1, dH_fem, &dH_fdm); //phgExportVTK(g, "dH_diff.vtk", dH_fdm, NULL); *///phgExportVTK(g, "dH_fdm.vtk", ns->dH, NULL); get_moved_coord(ns, tstep); phgExportVTK(g, "dH_fem2.vtk", NULL); move_mesh(ns); //phgExportVTK(g, "moved_geo.vtk", NULL); //check_height(g, gL); } else { /* Structed layered mesh */ struct_mesh_update(ns, tstep, Time); } phgDofGradient(u[1], &gradu[1], NULL, "gradu_{n+1}"); phgDofGradient(u[0], &gradu[0], NULL, "gradu_{n}"); } else { phgPrintf("Mesh not moved.\n"); } phgPrintf("\n-----------------------------------------\n"); phgPrintf(" dH: [%4.2E, %4.2E]\n", phgDofMinValVec(ns->dH), phgDofMaxValVec(ns->dH)); phgPrintf("------------------------------------------\n\n"); FLOAT ice_volume = get_ice_volume(g); //DOF *u_P1 = phgDofCopy(u[1], NULL, DOF_P1, NULL); //phgExportVTK(g, "u_P1.vtk", u_P1, NULL); #if 1 //INT dH_convergence = check_surf_convergence(ns, dH_last); //if (dH_convergence == 1) FLOAT dVdt = fabs(ice_volume-ice_volume_last)/ice_volume/dt[0]; if (Time > 10 && dVdt < ns_params->s_tol) { phgPrintf("-----------------------------------------------------\n"); phgPrintf("The ice domain reaches a steady state! Model stops! dVdt %e", dVdt); phgPrintf("\n-----------------------------------------------------\n"); break; } else { phgPrintf("\n--------------------------------------------------------------\n"); phgPrintf("The ice domain is still in an unsteady state! Model continues! dVdt %e\n", dVdt); phgPrintf("---------------------------------------------------------------\n\n"); } #endif update_grounding_line(ns, tstep); // after we update the geometry, we need to check the ice shelf mask again /* ------------------------------------------------------------ * * Output solution * * ------------------------------------------------------------ */ if (tstep % ns_params->step_span == 0) { //ice_monitor(ns, tstep); #if 1 /* Temp check */ phgPrintf(" Output solution to VTK"); /* phgExportEnsightT(g, OUTPUT_DIR "/ins_" NS_PROBLEM , Time, tstep, */ /* u[1], p[1], T[1], ns->depth_P1, */ /* NULL); /\* ensight *\/ */ DOF *u_P1 = phgDofCopy(u[1], NULL, DOF_P1, NULL); //phgExportVTK(g,"results.vtk",u_P1,p[1],NULL); sprintf(vtk_file, dH_OUTPUT_DIR "ice_%05d.vtk", tstep); phgExportVTK(g, vtk_file, u_P1, NULL); phgDofFree(&u_P1); sprintf(vtk_file, dH_OUTPUT_DIR "dH_%05d.vtk", tstep); phgExportVTK(g, vtk_file, ns->dH, NULL); sprintf(vtk_file, dH_OUTPUT_DIR "stress_%05d.vtk", tstep); phgExportVTK(g, vtk_file, ns->stress, NULL); //sprintf(vtk_file, OUTPUT_DIR "mask_%05d.vtk", tstep); //phgExportVTK(g, vtk_file, ns->mask_bot, NULL); elapsed_time(g, TRUE, 0.); #endif /* Save coord data */ #if 1 if (tstep % ns_params->step_span_resume == 0) { { //sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", tstep); sprintf(data_Crd, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat.Crd", 2); assert(ns->coord->type == DOF_P1); save_dof_data3(g, ns->coord, data_Crd); } if (ns_params->record) { /* save dof data for time step {n}, {n+1} */ //sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep - 1); sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 1); DATA_FILE_SURFIX; save_dof_data3(g, u[0], data_u); save_dof_data3(g, p[0], data_p); save_dof_data3(g, T[0], data_T); phgPrintf(" Save u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n", tstep - 1, DofGetDataCountGlobal(u[0]), phgDofNormL2(u[0]), tstep - 1, DofGetDataCountGlobal(p[0]), phgDofNormL2(p[0])); phgPrintf(" Save T_{%5d}[%8d]:%24.12E\n", tstep - 1, DofGetDataCountGlobal(T[0]), phgDofNormL2(T[0])); DOF_SCALE(u[0], "save"); DOF_SCALE(p[0], "save"); DOF_SCALE(T[0], "save"); DOF_SCALE(gradu[0], "save"); //sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", tstep); sprintf(data_file, OUTPUT_DIR "/ins_" NS_PROBLEM "_%05d.dat", 2); DATA_FILE_SURFIX; save_dof_data3(g, u[1], data_u); save_dof_data3(g, p[1], data_p); save_dof_data3(g, T[1], data_T); phgPrintf(" Save u_ {%5d}[%8d]:%24.12E p_ {%5d}[%8d]:%24.12E\n", tstep, DofGetDataCountGlobal(u[1]), phgDofNormL2(u[1]), tstep, DofGetDataCountGlobal(p[1]), phgDofNormL2(p[1])); phgPrintf(" Save T_{%5d}[%8d]:%24.12E\n", tstep, DofGetDataCountGlobal(T[1]), phgDofNormL2(T[1])); DOF_SCALE(u[1], "save"); DOF_SCALE(p[1], "save"); DOF_SCALE(T[1], "save"); DOF_SCALE(gradu[1], "save"); phgResumeLogUpdate(g, NULL, NULL, NULL, data_file); } /* end of record */ if (gL != NULL) { //check_height(g, gL); } sayHello("After record final solution data"); } #endif } //phgDofFree(&dH_last); /* clean up */ //phgDofFree(&ediv); //phgDofFree(&eu); //phgDofFree(&ep); //phgDofFree(&eT); /* ---------------------------------------------------------------------- * * Compute drag force FD * * ---------------------------------------------------------------------- * */ phgGetTime(tt1); phgPrintf(" total time usage of current time step: %lfs\n", (double)(tt1[2] - tt[2])); if (mem_peak > 1024 * (size_t)ns_params->mem_max * 1024) { phgPrintf("\n=======\nMem usage reach max, exit.\n"); break; } tstep++; #if 1 /* use time t^{n} */ dt[-1] = dt[0]; if (Time + dt[0] > time_end) dt[0] = time_end - Time; Time += dt[0]; setFuncTime(Time); #endif phgDofFree(&surf_bas->dof); phgDofFree(&ns->surf_bas->dof); } /* end of time advaning */ /* destroy line block */ //destroy_line_block(&ns->bk); /* destroy reused solver */ if (ns->solver_u != NULL) { if (ns_params->use_PCD) phgNSDestroyPc(ns); phgNSDestroySolverU(ns); } phgNSFinalize(&ns); phgFreeGrid(&g); phgFinalize(); phgFree(ns_params); return 0; }
int main(int argc, char *argv[]) { MAT *A, *B; VEC *U = NULL, *x; FLOAT *C; SOLVER *solver, *solver1 = NULL; INT i, j, k, n, *pvt, N = 1000, K = 2; char *main_opts = NULL, *sub_opts = NULL; phgOptionsRegisterInt("-n", "N value", &N); phgOptionsRegisterInt("-k", "K value", &K); phgOptionsRegisterString("-main_solver_opts", "Options for the main solver", &main_opts); phgOptionsRegisterString("-sub_solver_opts", "Options for the subsolver", &sub_opts); /* a direct solver is preferable for the sparse matrix */ phgOptionsPreset("-solver mumps"); phgInit(&argc, &argv); phgPrintf( "----------------------------------------------------------------------------\n" "This code solves (A+UU^t)x=b using the Sherman-Morrison-Woodbury formula.\n" "Note: may use the following to disable use of the Sherman-Morrison-Woodbury\n" "algorithm and change to the default solver instead:\n" " -preonly_pc_type solver -preonly_pc_opts \"-solver_maxit 2000\"\n" "----------------------------------------------------------------------------\n" ); phgPrintf("Generating the linear system: N = %"dFMT", K = %"dFMT"\n", N, K); /* A is a distributed NxN SPD tridiagonal matrix (A = [-1, 2, -1]) */ n = N / phgNProcs + (phgRank < (N % phgNProcs) ? 1 : 0); A = phgMatCreate(phgComm, n, N); phgPrintf(" Generating matrix A.\n"); for (i = 0; i < n; i++) { /* diagonal */ phgMatAddEntry(A, i, i, 2.0); /* diagonal - 1 */ if (i > 0) phgMatAddEntry(A, i, i - 1, -1.0); else if (phgRank > 0) phgMatAddLGEntry(A, i, A->rmap->partition[phgRank] - 1, -1.0); /* diagonal + 1 */ if (i < n - 1) phgMatAddEntry(A, i, i + 1, -1.0); else if (phgRank < phgNProcs - 1) phgMatAddLGEntry(A, i, A->rmap->partition[phgRank] + n, -1.0); } phgMatAssemble(A); /* U is a K-component vector */ U = phgMapCreateVec(A->rmap, K); phgVecRandomize(U, 123); /* solver1 is the solver for A */ phgOptionsPush(); phgOptionsSetOptions(sub_opts); solver1 = phgMat2Solver(SOLVER_DEFAULT, A); phgOptionsPop(); /* x is a scratch vector */ x = phgMapCreateVec(A->rmap, 1); /* C is a KxK dense matrix, pvt is an integer array, they store the LU * factorization of (I + U^t*inv(A)*U) */ phgPrintf(" Generating the dense matrix I+U^t*inv(A)*U.\n"); C = phgCalloc(K * K, sizeof(*C)); pvt = phgAlloc(K * sizeof(*pvt)); for (i = 0; i < K; i++) { for (j = 0; j < n; j++) { solver1->rhs->data[j] = U->data[i * n + j]; x->data[j] = 0.0; } solver1->rhs->assembled = TRUE; phgSolverVecSolve(solver1, FALSE, x); for (j = 0; j < K; j++) for (k = 0; k < n; k++) C[i * K + j] += U->data[j * n + k] * x->data[k]; } #if USE_MPI if (U->map->nprocs > 1) { FLOAT *tmp = phgAlloc(K * K * sizeof(*tmp)); MPI_Allreduce(C, tmp, K * K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm); phgFree(C); C = tmp; } #endif /* USE_MPI */ for (i = 0; i < K; i++) C[i * K + i] += 1.0; phgPrintf(" Factorizing the dense matrix I+U^t*inv(A)*U.\n"); phgSolverDenseLU(K, C, pvt); /* B is a matrix-free matrix representing A + U*U^t, B->mv_data is used * to pass A, U, solver1, C and pvt to callback functions */ B = phgMapCreateMatrixFreeMat(A->rmap, A->cmap, funcB, /* arguments carried over to CB functions */ A, U, solver1, C, pvt, NULL); /* solver is a PreOnly solver for B whose pc_proc is set to sherman(). * * Note: can also use pcg, gmres, or petsc for this solver, in this case * the solution obtained with the Sherman-Morisson formula is iteratively * refined. */ phgOptionsPush(); phgOptionsSetOptions("-solver preonly"); phgOptionsSetOptions(main_opts); solver = phgMat2Solver(SOLVER_DEFAULT, B); phgSolverSetPC(solver, solver, sherman); phgOptionsPop(); for (i = 0; i < n; i++) x->data[i] = 1.0; phgMatVec(MAT_OP_N, 1.0, B, x, 0.0, &solver->rhs); phgPrintf("Solving the linear system.\n"); /* reset initial solution to zero */ memset(x->data, 0, n * sizeof(*x->data)); phgSolverVecSolve(solver, TRUE, x); for (i = 0; i < n; i++) solver->rhs->data[i] = 1.0; phgVecAXPBY(-1.0, solver->rhs, 1.0, &x); phgPrintf("Checking the result: |x - x_exact| / |x_exact| = %lg\n", (double)phgVecNorm2(x, 0, NULL) / sqrt((double)N)); phgSolverDestroy(&solver); phgSolverDestroy(&solver1); phgMatDestroy(&A); phgMatDestroy(&B); phgVecDestroy(&U); phgVecDestroy(&x); phgFree(C); phgFree(pvt); phgFinalize(); return 0; }
int main(int argc, char *argv[]) { GRID *g; DOF *u_h; MAT *A, *A0, *B; MAP *map; INT i; size_t nnz, mem, mem_peak; VEC *x, *y0, *y1, *y2; double t0, t1, dnz, dnz1, mflops, mop; char *fn = "../test/cube.dat"; FLOAT mem_max = 300; INT refine = 0; phgOptionsRegisterFilename("-mesh_file", "Mesh file", (char **)&fn); phgOptionsRegisterInt("-loop_count", "Loop count", &loop_count); phgOptionsRegisterInt("-refine", "Refinement level", &refine); phgOptionsRegisterFloat("-mem_max", "Maximum memory", &mem_max); phgInit(&argc, &argv); g = phgNewGrid(-1); if (!phgImport(g, fn, FALSE)) phgError(1, "can't read file \"%s\".\n", fn); phgRefineAllElements(g, refine); u_h = phgDofNew(g, DOF_DEFAULT, 1, "u_h", DofNoAction); while (TRUE) { phgPrintf("\n"); if (phgBalanceGrid(g, 1.2, 1, NULL, 0.)) phgPrintf("Repartition mesh, %d submeshes, load imbalance: %lg\n", g->nprocs, (double)g->lif); map = phgMapCreate(u_h, NULL); A = phgMapCreateMat(map, map); A->handle_bdry_eqns = TRUE; build_matrix(A, u_h); phgMatAssemble(A); /* Note: A is unsymmetric (A' != A) if boundary entries not removed */ phgMatRemoveBoundaryEntries(A); #if 0 /* test block matrix operation */ A0 = phgMatCreateBlockMatrix(g->comm, 1, 1, &A, NULL); #else A0 = A; #endif phgPrintf("%d DOF, %d elems, %d submeshes, matrix size: %d, LIF: %lg\n", DofGetDataCountGlobal(u_h), g->nleaf_global, g->nprocs, A->rmap->nglobal, (double)g->lif); /* test PHG mat-vec multiply */ x = phgMapCreateVec(A->cmap, 1); y1 = phgMapCreateVec(A->rmap, 1); phgVecRandomize(x, 123); phgMatVec(MAT_OP_N, 1.0, A0, x, 0.0, &y1); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); for (i = 0; i < loop_count; i++) { phgMatVec(MAT_OP_N, 1.0, A0, x, 0.0, &y1); } t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); y0 = phgVecCopy(y1, NULL); nnz = A->nnz_d + A->nnz_o; #if USE_MPI dnz1 = nnz; MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm); #else dnz = nnz; #endif mop = loop_count * (dnz + dnz - A->rmap->nlocal) * 1e-6; phgPrintf("\n"); t1 -= t0; phgPrintf(" PHG: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF)\n", t1, dnz, mop / (t1 == 0 ? 1. : t1), mflops); /* test trans(A)*x */ phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); for (i = 0; i < loop_count; i++) { phgMatVec(MAT_OP_T, 1.0, A0, x, 0.0, &y1); } t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); t1 -= t0; phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), " "err: %le\n", t1, dnz, mop / (t1 == 0 ? 1. : t1), mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL)); /* time A * trans(A) */ phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); B = phgMatMat(MAT_OP_N, MAT_OP_N, 1.0, A, A, 0.0, NULL); t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); nnz = B->nnz_d + B->nnz_o; #if USE_MPI dnz1 = nnz; MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm); #else dnz = nnz; #endif /* compare B*x <--> A*A*x */ y2 = phgMatVec(MAT_OP_N, 1.0, B, x, 0.0, NULL); phgMatVec(MAT_OP_N, 1.0, A0, y0, 0.0, &y1); phgMatDestroy(&B); t1 -= t0; phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n", t1, dnz, mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL)); #if USE_PETSC { Mat ma, mb; MatInfo info; Vec va, vb, vc; PetscScalar *vec; ma = phgPetscCreateMatAIJ(A); MatGetVecs(ma, PETSC_NULL, &va); VecDuplicate(va, &vb); VecGetArray(va, &vec); memcpy(vec, x->data, x->map->nlocal * sizeof(*vec)); VecRestoreArray(va, &vec); MatMult(ma, va, vb); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); for (i = 0; i < loop_count; i++) { MatMult(ma, va, vb); } t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); VecGetArray(vb, &vec); memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec)); VecRestoreArray(vb, &vec); MatGetInfo(ma, MAT_GLOBAL_SUM, &info); /*phgPrintf(" --------------------------------------------" "-------------------------\n");*/ phgPrintf("\n"); t1 -= t0; dnz = info.nz_used; phgPrintf(" PETSc: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), " "err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL)); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); for (i = 0; i < loop_count; i++) { MatMultTranspose(ma, va, vb); } t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); VecGetArray(vb, &vec); memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec)); VecRestoreArray(vb, &vec); t1 -= t0; phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), " "err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL)); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); MatMatMult(ma, ma, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &mb); t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); t1 -= t0; MatGetInfo(mb, MAT_GLOBAL_SUM, &info); dnz = info.nz_used; VecDuplicate(va, &vc); /* compare B*x <--> A*A*x */ MatMult(ma, vb, vc); MatMult(mb, va, vb); VecGetArray(vb, &vec); memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec)); VecRestoreArray(vb, &vec); VecGetArray(vc, &vec); memcpy(y2->data, vec, x->map->nlocal * sizeof(*vec)); VecRestoreArray(vc, &vec); phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n", t1, dnz, mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL)); phgPetscMatDestroy(&mb); phgPetscMatDestroy(&ma); phgPetscVecDestroy(&va); phgPetscVecDestroy(&vb); phgPetscVecDestroy(&vc); } #endif /* USE_PETSC */ #if USE_HYPRE { HYPRE_IJMatrix ma; HYPRE_IJVector va, vb, vc; HYPRE_ParCSRMatrix par_ma; hypre_ParCSRMatrix *par_mb; HYPRE_ParVector par_va, par_vb, par_vc; HYPRE_Int offset, *ni, start, end; assert(sizeof(INT)==sizeof(int) && sizeof(FLOAT)==sizeof(double)); setup_hypre_mat(A, &ma); ni = phgAlloc(2 * A->rmap->nlocal * sizeof(*ni)); offset = A->cmap->partition[A->cmap->rank]; for (i = 0; i < A->rmap->nlocal; i++) ni[i] = i + offset; HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1, &va); HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1, &vb); HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1, &vc); HYPRE_IJVectorSetObjectType(va, HYPRE_PARCSR); HYPRE_IJVectorSetObjectType(vb, HYPRE_PARCSR); HYPRE_IJVectorSetObjectType(vc, HYPRE_PARCSR); HYPRE_IJVectorSetMaxOffProcElmts(va, 0); HYPRE_IJVectorSetMaxOffProcElmts(vb, 0); HYPRE_IJVectorSetMaxOffProcElmts(vc, 0); HYPRE_IJVectorInitialize(va); HYPRE_IJVectorInitialize(vb); HYPRE_IJVectorInitialize(vc); HYPRE_IJMatrixGetObject(ma, (void **)(void *)&par_ma); HYPRE_IJVectorGetObject(va, (void **)(void *)&par_va); HYPRE_IJVectorGetObject(vb, (void **)(void *)&par_vb); HYPRE_IJVectorGetObject(vc, (void **)(void *)&par_vc); HYPRE_IJVectorSetValues(va, A->cmap->nlocal, ni, (double *)x->data); HYPRE_IJVectorAssemble(va); HYPRE_IJVectorAssemble(vb); HYPRE_IJVectorAssemble(vc); HYPRE_IJMatrixGetRowCounts(ma, A->cmap->nlocal, ni, ni + A->rmap->nlocal); for (i = 0, nnz = 0; i < A->rmap->nlocal; i++) nnz += ni[A->rmap->nlocal + i]; #if USE_MPI dnz1 = nnz; MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm); #else dnz = nnz; #endif HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_va, 0.0, par_vb); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); for (i = 0; i < loop_count; i++) { HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_va, 0.0, par_vb); } t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data); /*phgPrintf(" --------------------------------------------" "-------------------------\n");*/ phgPrintf("\n"); t1 -= t0; phgPrintf(" HYPRE: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), " "err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL)); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); for (i = 0; i < loop_count; i++) { HYPRE_ParCSRMatrixMatvecT(1.0, par_ma, par_va, 0.0, par_vb); } t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data); t1 -= t0; phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), " "err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL)); phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */ t0 = phgGetTime(NULL); /* Note: 'HYPRE_ParCSRMatrix' is currently typedef'ed to * 'hypre_ParCSRMatrix *' */ par_mb = hypre_ParMatmul((hypre_ParCSRMatrix *)par_ma, (hypre_ParCSRMatrix *)par_ma); t1 = phgGetTime(NULL); mflops = phgPerfGetMflops(g, NULL, NULL); start = hypre_ParCSRMatrixFirstRowIndex(par_mb); end = hypre_ParCSRMatrixLastRowIndex(par_mb) + 1; for (i = start, nnz = 0; i < end; i++) { HYPRE_Int ncols; hypre_ParCSRMatrixGetRow(par_mb, i, &ncols, NULL, NULL); hypre_ParCSRMatrixRestoreRow(par_mb, i, &ncols, NULL, NULL); nnz += ncols; } #if USE_MPI dnz1 = nnz; MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm); #else dnz = nnz; #endif /* compare B*x <--> A*A*x */ HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_vb, 0.0, par_vc); HYPRE_ParCSRMatrixMatvec(1.0, (void *)par_mb, par_va, 0.0, par_vb); HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data); HYPRE_IJVectorGetValues(vc, A->rmap->nlocal, ni, (double*)y2->data); hypre_ParCSRMatrixDestroy((par_mb)); t1 -= t0; phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n", t1, dnz, mflops, (double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL)); phgFree(ni); HYPRE_IJMatrixDestroy(ma); HYPRE_IJVectorDestroy(va); HYPRE_IJVectorDestroy(vb); HYPRE_IJVectorDestroy(vc); } #endif /* USE_HYPRE */ if (A0 != A) phgMatDestroy(&A0); #if 0 if (A->rmap->nglobal > 1000) { VEC *v = phgMapCreateVec(A->rmap, 3); for (i = 0; i < v->map->nlocal; i++) { v->data[i + 0 * v->map->nlocal] = 1 * (i + v->map->partition[g->rank]); v->data[i + 1 * v->map->nlocal] = 2 * (i + v->map->partition[g->rank]); v->data[i + 2 * v->map->nlocal] = 3 * (i + v->map->partition[g->rank]); } phgMatDumpMATLAB(A, "A", "A.m"); phgVecDumpMATLAB(v, "v", "v.m"); phgFinalize(); exit(0); } #endif phgMatDestroy(&A); phgVecDestroy(&x); phgVecDestroy(&y0); phgVecDestroy(&y1); phgVecDestroy(&y2); phgMapDestroy(&map); mem = phgMemoryUsage(g, &mem_peak); dnz = mem / (1024.0 * 1024.0); dnz1 = mem_peak / (1024.0 * 1024.0); /*phgPrintf(" --------------------------------------------" "-------------------------\n");*/ phgPrintf("\n"); phgPrintf(" Memory: current %0.4lgMB, peak %0.4lgMB\n", dnz, dnz1); #if 0 { static int loop_count = 0; if (++loop_count == 4) break; } #endif if (mem_peak > 1024 * (size_t)1024 * mem_max) break; phgRefineAllElements(g, 1); } phgDofFree(&u_h); phgFreeGrid(&g); phgFinalize(); return 0; }