Beispiel #1
0
static void 
getBdryTypes(MovingMesh *mmesh) 
{
    GRID *g = _m->g;
    SIMPLEX *e;
    FILE *fp;
    char token[1024]; 
    int i, n, s;
    int max_bdry_type = sizeof(BTYPE) * 8;
    
    Unused(s);
    Unused(e);
    Unused(g);

    /* Read in boundary faces info from file */
    fp = fopen(_mp->fn_bdry, "r");

    /* Header */
    if (!get_token(fp, token) || strcasecmp(token, "BoundaryFaces")) 
	READ_ERROR;

    if (!get_token(fp, token))
	READ_ERROR;
    n = atoi(token);
    phgInfo(2, "number of bdry faces: %d\n", n);
    _mb->move_bdry_mask = 0;
    _mb->n_bdry = max_bdry_type;
    _mb->normal = phgCalloc(max_bdry_type * Dim, sizeof(*_mb->normal));
    _mb->b = phgCalloc(max_bdry_type, sizeof(*_mb->b));
    
    /* Read in noraml and projection */
    for (i = 0; i < n; i++) {
	int type;
	READ_NUMBER;
	type = atoi(token) - 1;
	if (!get_token(fp, token))
	    READ_ERROR;
	_mb->normal[type*Dim + 0] = atof(token);
	if (!get_token(fp, token))
	    READ_ERROR;
	_mb->normal[type*Dim + 1] = atof(token);
	if (!get_token(fp, token))
	    READ_ERROR;
	_mb->normal[type*Dim + 2] = atof(token);
	if (!get_token(fp, token))
	    READ_ERROR;
	_mb->b[type] = atof(token);
	_mb->move_bdry_mask |= btype_values[type];
    }

    /* End of read */
    if (!get_token(fp, token) || strcasecmp(token, "End"))
	READ_ERROR;
    fclose(fp);

    return;
}
/* PC_PROC function which applies the Sherman-Morrison-Woodbury formula */
static void
sherman(void *ctx, VEC *b0, VEC **x0)
{
    SOLVER *solver = ctx;
    VEC *U = ((void **)solver->mat->mv_data)[1];
    SOLVER *solver1 = ((void **)solver->mat->mv_data)[2];
    FLOAT *C = ((void **)solver->mat->mv_data)[3];
    INT *pvt = ((void **)solver->mat->mv_data)[4];

    VEC *x = *x0, *y;
    INT i, j, n = U->map->nlocal, K = U->nvec;
    FLOAT *z;

    phgPrintf("Note: applying the Sherman-Morrison-Woodbury formula.\n");

    /* x := inv(A) * b */
#if 0
    memset(x->data, 0, n * sizeof(*x->data));
#endif
    z = solver1->rhs->data; 	/* save solver1->rhs->data */
    solver1->rhs->data = b0->data;
    solver1->rhs->assembled = TRUE;
    phgSolverVecSolve(solver1, FALSE, x);
    solver1->rhs->data = z; 	/* restore solver1->rhs->data */

    if (K == 0)
	return;

    /* z := U^t * x */
    z = phgCalloc(K, sizeof(*z));
    for (j = 0; j < K; j++)
	for (i = 0; i < n; i++)
	    z[j] += U->data[j * n + i] * x->data[i];
#if USE_MPI
    if (U->map->nprocs > 1) {
	FLOAT *tmp = phgAlloc(K * sizeof(*tmp));
	MPI_Allreduce(z, tmp, K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm);
	phgFree(z);
	z = tmp;
    }
#endif	/* USE_MPI */

    /* z := C*z */
    phgSolverDenseSV(K, C, pvt, 1, z);

    /* y := inv(A) * (U*z) */
    memset(solver1->rhs->data, 0, n * sizeof(*solver1->rhs->data));
    for (i = 0; i < n; i++)
	for (j = 0; j < K; j++)
	    solver1->rhs->data[i] += U->data[j * n + i] * z[j];
    y = phgMapCreateVec(U->map, 1);
    phgSolverVecSolve(solver1, FALSE, y);
    phgFree(z);

    /* x -= y */
    for (i = 0; i < n; i++)
	x->data[i] -= y->data[i];

    phgVecDestroy(&y);
}
/* callback function which computes A+UU^t */
static int
funcB(MAT_OP op, MAT *B, VEC *x, VEC *y)
/* computes y = op(A) * x */
{
    MAT *A = ((void **)B->mv_data)[0];
    VEC *U = ((void **)B->mv_data)[1];
    INT i, j, k, n = U->map->nlocal, K = U->nvec;

    /* compute y = op(A)*x */
    phgMatVec(op, 1.0, A, x, 0.0, &y);

    if (K == 0)
	return 0;

    if (op != MAT_OP_D) {
	/* compute y += U*U^t*x */
	FLOAT *z = phgCalloc(K, sizeof(*z));
	/* z = U^t*x */
	for (j = 0; j < K; j++)
	    for (i = 0; i < n; i++)
		z[j] += U->data[j * n + i] * x->data[i];
#if USE_MPI
	if (U->map->nprocs > 1) {
	    FLOAT *tmp = phgCalloc(K, sizeof(*tmp));
	    MPI_Allreduce(z, tmp, K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm);
	    phgFree(z);
	    z = tmp;
	}
#endif	/* USE_MPI */
	/* y += U*z */
	for (i = 0; i < n; i++)
	    for (j = 0; j < K; j++)
		y->data[i] += U->data[j * n + i] * z[j];
	phgFree(z);
	return 0;
    }

    for (i = 0; i < n; i++) {
	FLOAT d = 0.0;
	for (j = 0; j < K; j++)
	    for (k = 0; k < K; k++)
		d += U->data[j * n + i] * U->data[k * n + i];
	y->data[i] += d * x->data[i];
    }

    return 0;
}
Beispiel #4
0
/*
 *  Moving mesh initialization, following jobs are done:
 *  1. create interior vertex map and boundary vertex map,
 *  2. create a logical mesh.
 *  */
MovingMesh *
phgMovingMeshCreate(GRID *g, PARAMS *params)
{
    MovingMesh *mmesh;
    INT i, k;
    FLOAT *v;
    
    _m = phgCalloc(1, sizeof(*_m));
    _m->params = params;
    _m->g = g;

    _m->logical_node = 
	phgDofNew(g, DOF_P1, 3, "logical node coordinates", DofNoAction);
    _m->logical_node->DB_mask[0] = MM_CONSTR0;
    _m->logical_node->DB_mask[1] = MM_CONSTR1;
    _m->logical_node->DB_mask[2] = MM_CONSTR2;

    _m->logical_move = 
	phgDofNew(g, DOF_P1, 3, "logical move direction", DofNoAction);
    _m->move = 
	phgDofNew(g, DOF_P1, 3, "node move direction", DofNoAction);
    _m->monitor = 
	phgDofNew(g, DOF_P0, 1, "monitor", DofNoAction);

    /* P1 bases */
    _m->phi = 
	phgDofNew(g, DOF_P1, 1, "P1 bases", DofNoAction);
    _m->phi->DB_mask[0] = 0;
    _m->map = phgMapCreate(_m->phi, NULL);

    /* Boundary types of verties */
    getBdryTypes(_m);

    /* get logical mesh
     * Note: here logical mesh is identical to original mesh*/
    v = _m->logical_node->data;
    for (i = 0; i < g->nvert; i++) {
	if (!(g->types_vert[i] == UNREFERENCED)) {
	    for (k = 0; k < Dim; k++)
		v[i*Dim + k] = g->verts[i][k];
	}
    } /* Point loop: local */
    DOF_SCALE(_m->logical_node);

    return _m;
}
Beispiel #5
0
DOF *
phgDofCopy_(DOF *src, DOF **dest_ptr, DOF_TYPE *newtype,
	    const char *name, const char *srcfile, int srcline)
/* returns a new DOF whose type is 'newtype', and whose values are evaluated
 * using 'src' (a copy of src). */
{
    GRID *g = src->g;
    SIMPLEX *e;
    FLOAT w = 1.0, *basvalues = NULL, *a, *d, *buffer = NULL;
    int i, k, dim, dim_new, count = 0, nvalues, nd;
    INT n;
    DOF *wgts = NULL;
    DOF *dest = (dest_ptr == NULL ? NULL : *dest_ptr);
    BYTE *flags0, *flags;
    char *auto_name;
    DOF_TYPE *oldtype;
    BOOLEAN basflag = FALSE;

    MagicCheck(DOF, dest)

    if (dest != NULL && newtype != NULL && dest->type != newtype) {
	phgDofFree(&dest);
	dest = NULL;
    }

    dim = DofDim(src);
    /* the name of dest */
    if ((auto_name = (void *)name) == NULL) {
#if 0
	auto_name = phgAlloc(strlen(src->name) + 8 + 1);
	sprintf(auto_name, "copy of %s", src->name);
#else
	auto_name = strdup(src->name);
#endif
    }

    if (dest == NULL) {
	if (newtype == NULL)
	    newtype = src->type;
	dim_new = (newtype == NULL ? DofTypeDim(src) : newtype->dim);
	assert(dim % dim_new == 0);
	dest = phgDofNew_(g, newtype, newtype == NULL ? src->hp : NULL,
			  dim / dim_new, auto_name, DofNoAction,
			  srcfile, srcline);
	if (dest_ptr != NULL)
	    *dest_ptr = dest;
    }
    else {
	assert(dim == DofDim(dest));
	phgFree(dest->name); dest->name = strdup(auto_name);
	dest->srcfile = srcfile;
	dest->srcline = srcline;
	phgFree(dest->cache_func); dest->cache_func = NULL;
	newtype = dest->type;
	if (!SpecialDofType(newtype))
	    memset(dest->data, 0, DofGetDataCount(dest) * sizeof(*dest->data));
    }
    if (auto_name != name)
	phgFree(auto_name);

    phgDofClearCache(NULL, dest, NULL, NULL, FALSE);

    dest->DB_mask = src->DB_mask;
    if (src->DB_masks != NULL) {
	if (dest->DB_masks == NULL)
	    dest->DB_masks = phgAlloc(dest->dim * sizeof(*dest->DB_masks));
	memcpy(dest->DB_masks, src->DB_masks, dest->dim * sizeof(*dest->DB_masks));
    }
    dest->invariant = src->invariant;

    if (SpecialDofType(newtype)) {
	assert(newtype == src->type);
	dest->userfunc = src->userfunc;
	dest->userfunc_lambda = src->userfunc_lambda;
	if (newtype == DOF_CONSTANT)
	    memcpy(dest->data, src->data, dim * sizeof(*dest->data));
	return dest;
    }

    if ((newtype != NULL && newtype == src->type) ||
	(newtype == NULL && src->hp == dest->hp)) {
	/* simply duplicate the data */
	size_t size = DofGetDataCount(dest);
	if (size > 0)
	    memcpy(dest->data, src->data, sizeof(FLOAT) * size);
	dest->userfunc = src->userfunc;
	dest->userfunc_lambda = src->userfunc_lambda;
	return dest;
    }

    if (src->type == DOF_ANALYTIC) {
	if (src->userfunc_lambda != NULL)
	    phgDofSetDataByLambdaFunction(dest, src->userfunc_lambda);
	else
	    phgDofSetDataByFunction(dest, src->userfunc);
	return dest;
    }

    if (newtype != NULL && newtype->BasFuncs == NULL)
	phgError(1, "phgDofCopy: basis funcs for DOF type \"%s\" undefined.\n",
		 newtype->name);

    dest->userfunc = src->userfunc;
    dest->userfunc_lambda = src->userfunc_lambda;

    oldtype = src->type;
    if (oldtype == NULL)
	oldtype = src->hp->info->types[src->hp->info->min_order];

    if (!SpecialDofType(oldtype) && newtype != NULL && newtype->points != NULL
	&& !DofIsHP(src)) {
	basflag = TRUE;
	count = oldtype->nbas * oldtype->dim;
	basvalues = phgAlloc(newtype->nbas * count * sizeof(*basvalues));

	if (oldtype->invariant == TRUE)
	    get_bas_funcs(src, dest, src->g->roots, basvalues);
    }

    if (newtype == NULL)
	newtype = dest->hp->info->types[dest->hp->max_order];

    flags0 = phgCalloc((newtype->np_vert > 0 ? g->nvert : 0)
		       + (newtype->np_edge > 0 ? g->nedge : 0)
		       + (newtype->np_face > 0 ? g->nface : 0),
		       sizeof(*flags0));

    if (!SpecialDofType(oldtype) && oldtype->continuity < 0) {
	static DOF_TYPE DOF_WGTS = {DofCache,
	    "Weights", NULL, NULL, NULL, NULL, NULL, NULL, NULL,
	    phgDofInitFuncPoint, NULL, NULL, NULL, FE_None,
	    FALSE, FALSE, -1, 0, 0, -1, 1, 0, 0, 0, 0
	};
	DOF_WGTS.np_vert = (newtype->np_vert > 0) ? 1 : 0;
	DOF_WGTS.np_edge = (newtype->np_edge > 0) ? 1 : 0;
	DOF_WGTS.np_face = (newtype->np_face > 0) ? 1 : 0;
	DOF_WGTS.nbas = DOF_WGTS.np_vert * NVert + DOF_WGTS.np_edge * NEdge +
	    DOF_WGTS.np_face * NFace;
	if (DOF_WGTS.nbas > 0) {
	    /* Other cases will be implemented later when really needed */
	    wgts = phgDofNew(g, &DOF_WGTS, 1, "weights", DofNoAction);
	    phgDofSetDataByValue(wgts, 0.0);
	    phgDofSetDataByValue(dest, 0.0);
	    /* allocate buffer for storing weighted vertex/edge/face data */
	    if ((n = DofGetDataCount(dest) - DofGetElementDataCount(dest)) > 0)
		buffer = phgCalloc(n, sizeof(*buffer));
	}
    }

    nvalues = dest->dim;
    cache_dof = src;
    bas_count = count;
    ForAllElements(g, e) {
	if (wgts != NULL) {
#if 0
	    /* use element volume as weight */
	    w = phgGeomGetVolume(g, e);
#else
	    /* use 1 as weight */
	    w = 1.0;
#endif
	}

	if (basflag && oldtype->invariant == FALSE)
	    get_bas_funcs(src, dest, e, basvalues);
	bas = basvalues;
	flags = flags0;

	if (DofIsHP(dest))
	    newtype = dest->hp->info->types[dest->hp->elem_order[e->index]];

	if (newtype->np_vert > 0) {
	    nd = nvalues * newtype->np_vert;
	    for (k = 0; k < NVert; k++) {
		if (flags[n = e->verts[k]] && wgts == NULL) {
		    /* note: count==0 and bas==NULL for variable order DOF */
		    bas += count * newtype->np_vert;
		    continue;
		}
		flags[n] = TRUE;
		a = DofVertexData(dest, n);
		newtype->InitFunc(dest, e, VERTEX, k, NULL, func, NULL, a,
					NULL);
		if (wgts != NULL) {
		    d = buffer + (a - DofData(dest));
		    for (i = 0; i < nd; i++)
			*(d++) += *(a++) * w;
		    *DofVertexData(wgts, n) += w;
		}
	    }
	    flags += g->nvert;
	}

	if (newtype->np_edge > 0) {
	    nd = nvalues * newtype->np_edge;
	    for (k = 0; k < NEdge; k++) {
		if (flags[n = e->edges[k]] && wgts == NULL) {
		    /* note: count==0 and bas==NULL for variable order DOF */
		    bas += count * newtype->np_edge;
		    continue;
		}
		flags[n] = TRUE;
		a = DofEdgeData(dest, n);
		newtype->InitFunc(dest, e, EDGE, k, NULL, func, NULL, a,
					NULL);
		if (wgts != NULL) {
		    d = buffer + (a - DofData(dest));
		    if (DofIsHP(dest))
			nd = dest->dim * (dest->hp->edge_index[n + 1] -
					  dest->hp->edge_index[n]);
		    for (i = 0; i < nd; i++)
			*(d++) += *(a++) * w;
		    *DofEdgeData(wgts, n) += w;
		}
	    }
	    flags += g->nedge;
	}

	if (newtype->np_face > 0) {
	    nd = nvalues * newtype->np_face;
	    for (k = 0; k < NFace; k++) {
		if (flags[n = e->faces[k]] && wgts == NULL) {
		    /* note: count==0 and bas==NULL for variable order DOF */
		    bas += count * newtype->np_face;
		    continue;
		}
		flags[n] = TRUE;
		a = DofFaceData(dest, n);
		newtype->InitFunc(dest, e, FACE, k, NULL, func, NULL, a,
					NULL);
		if (wgts != NULL) {
		    d = buffer + (a - DofData(dest));
		    if (DofIsHP(dest))
			nd = dest->dim * (dest->hp->face_index[n + 1] -
						 dest->hp->face_index[n]);
		    for (i = 0; i < nd; i++)
			*(d++) += *(a++) * w;
		    *DofFaceData(wgts, n) += w;
		}
	    }
	}

	if (newtype->np_elem > 0) {
	    a = DofElementData(dest, e->index);
	    newtype->InitFunc(dest, e, ELEMENT, 0, NULL, func, NULL, a,
					NULL);
	}
    }

    phgFree(basvalues);
    phgFree(flags0);

    if (wgts == NULL)
	return dest;

    if ((n = DofGetDataCount(dest) - DofGetElementDataCount(dest)) > 0) {
	memcpy(DofData(dest), buffer, n * sizeof(*buffer));
	phgFree(buffer);
    }

    if (DofIsHP(dest))
	newtype = dest->hp->info->types[dest->hp->max_order];

    a = DofData(dest);
    d = DofData(wgts);

#if USE_MPI
    /* FIXME: directly interacting with the vector assembly code in solver.c
       is more efficient */
    if (g->nprocs > 1) {
	SOLVER *solver = phgSolverCreate(SOLVER_BUILTIN, dest, NULL);
	INT K = 0, I;
	int j;

	if (newtype->np_vert > 0) {
	    nd = dest->count_vert;
	    for (n = 0; n < g->nvert; n++, d++) {
		if (g->types_vert[n] == UNREFERENCED) {
		    K += nd;
		    a += nd;
		    continue;
		}
		for (j = 0; j < nd; j++, K++, a++) {
		    I = phgSolverMapD2L(solver, 0, K);
		    assert(I >= 0 && I < solver->mat->rmap->localsize);
		    phgSolverAddRHSEntry(solver, I, *a);
		    phgSolverAddMatrixEntry(solver, I, I, *d);
		}
	    }
	}

	if (newtype->np_edge > 0) {
	    nd = nvalues * newtype->np_edge;
	    for (n = 0; n < g->nedge; n++, d++) {
		if (DofIsHP(dest))
		    nd = dest->dim * (dest->hp->edge_index[n + 1] -
					     dest->hp->edge_index[n]);
		if (g->types_edge[n] == UNREFERENCED) {
		    K += nd;
		    a += nd;
		    continue;
		}
		for (j = 0; j < nd; j++, K++, a++) {
		    I = phgSolverMapD2L(solver, 0, K);
		    assert(I >= 0 && I < solver->mat->rmap->localsize);
		    phgSolverAddRHSEntry(solver, I, *a);
		    phgSolverAddMatrixEntry(solver, I, I, *d);
		}
	    }
	}

	if (newtype->np_face > 0) {
	    nd = nvalues * newtype->np_face;
	    for (n = 0; n < g->nface; n++, d++) {
		if (DofIsHP(dest))
		    nd = dest->dim * (dest->hp->face_index[n + 1] -
					     dest->hp->face_index[n]);
		if (g->types_face[n] == UNREFERENCED) {
		    K += nd;
		    a += nd;
		    continue;
		}
		for (j = 0; j < nd; j++, K++, a++) {
		    I = phgSolverMapD2L(solver, 0, K);
		    assert(I >= 0 && I < solver->mat->rmap->localsize);
		    phgSolverAddRHSEntry(solver, I, *a);
		    phgSolverAddMatrixEntry(solver, I, I, *d);
		}
	    }
	}

	if (newtype->np_elem > 0) {
	    nd = nvalues * newtype->np_elem;
	    for (n = 0; n < g->nelem; n++) {
		if (DofIsHP(dest))
		    nd = dest->dim * (dest->hp->elem_index[n + 1] -
					     dest->hp->elem_index[n]);
		if (g->types_elem[n] == UNREFERENCED) {
		    K += nd;
		    a += nd;
		    continue;
		}
		for (j = 0; j < nd; j++, K++, a++) {
		    I = phgSolverMapD2L(solver, 0, K);
		    assert(I >= 0 && I < solver->mat->rmap->localsize);
		    phgSolverAddRHSEntry(solver, I, *a);
		    phgSolverAddMatrixEntry(solver, I, I, 1.0);
		}
	    }
	}

	phgSolverSolve(solver, TRUE, dest, NULL);
	phgSolverDestroy(&solver);

	phgDofFree(&wgts);

	return dest;
    }
#endif

    if (newtype->np_vert > 0) {
	k = nvalues * newtype->np_vert;
	for (n = 0; n < g->nvert; n++) {
	    if ((w = *(d++)) == 0.0) {
		a += k;
	    }
	    else if (k == 1) {
		*(a++) /= w;
	    }
	    else {
		w = 1.0 / w;
		for (i = 0; i < k; i++)
		    *(a++) *= w;
	    }
	}
    }

    if (newtype->np_edge > 0) {
	k = nvalues * newtype->np_edge;
	for (n = 0; n < g->nedge; n++) {
	    if (DofIsHP(dest))
		k = dest->dim * (dest->hp->edge_index[n + 1] -
					dest->hp->edge_index[n]);
	    if ((w = *(d++)) == 0.0) {
		a += k;
	    }
	    else if (k == 1) {
		*(a++) /= w;
	    }
	    else {
		w = 1.0 / w;
		for (i = 0; i < k; i++)
		    *(a++) *= w;
	    }
	}
    }

    if (newtype->np_face > 0) {
	k = nvalues * newtype->np_face;
	for (n = 0; n < g->nface; n++) {
	    if (DofIsHP(dest))
		k = dest->dim * (dest->hp->face_index[n + 1] -
					dest->hp->face_index[n]);
	    if ((w = *(d++)) == 0.0) {
		a += k;
	    }
	    else if (k == 1) {
		*(a++) /= w;
	    }
	    else {
		w = 1.0 / w;
		for (i = 0; i < k; i++)
		    *(a++) *= w;
	    }
	}
    }

    phgDofFree(&wgts);

    return dest;
}
Beispiel #6
0
/* Get local bases for boundary vertex i,
 *  return num of constrained bases, 
 *  and noramlized direction of rotated bases.
 *  */
SURF_BAS *
get_surface_bases(GRID *g, DOF_TYPE *u_type)
{
    SIMPLEX *e;
    DOF *surf_dof = NULL, *norm_lat = NULL, *norm_bot = NULL;
    BOOLEAN *rotated = NULL;
    INT nrot = 0;
    surf_dof = phgDofNew(g, u_type, DDim, "Surf bases", DofNoAction);
    
    //norm_lat = phgDofNew(g, u_type, Dim, "Norm lateral", DofNoAction);
    //norm_bot = phgDofNew(g, u_type, Dim, "Norm Bottom", DofNoAction);
    //DOF *coord = phgDofNew(g, u_type, Dim, "coord", func_xyz_);
	DOF *avg_n = phgDofNew(g, DOF_P2, 3, "avg n", DofNoAction);
        get_avg_n(g, avg_n);
    
    rotated = phgCalloc(DofGetDataCount(surf_dof) / (DDim), sizeof(*rotated));
    SURF_BAS *surf_bas;

    surf_bas = phgCalloc(1, sizeof(*surf_bas));
    surf_bas->type = u_type;
    //surf_bas->dof = phgDofNew(g, u_type, DDim, "Surf bases", DofNoAction);//surf_dof;
    //surf_bas->dof = surf_dof;
    surf_bas->rotated = rotated;
    phgDofSetDataByValue(surf_dof, 0.);
    //phgDofSetDataByValue(surf_bas->dof, 0.);
    
    //phgDofSetDataByValue(norm_lat, -99.);
    //phgDofSetDataByValue(norm_bot, -99.);
    
        ForAllElements(g, e) {
	int s, ii, i, m, dof_i;
	int N = surf_dof->type->nbas;
	//int N = surf_bas->dof->type->nbas;
	FLOAT *avg_n_v;
	FLOAT normal[Dim];
	int  v[3];
	FLOAT norm;
	FLOAT norm_value_lat[N][Dim];
	FLOAT norm_value_bot[N][Dim];
	FLOAT bas_value[N][DDim],  H[Dim][Dim], c[Dim], 
	    bxyz[Dim][Dim] = {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}};

	phgDofGetElementData(surf_dof, e, &bas_value[0][0]);
	//phgDofGetElementData(surf_bas->dof, e, &bas_value[0][0]);
	
	for (s = 0; s < NFace; s++) { 
	    int nbas_face = NbasFace(surf_dof);
	    //int nbas_face = NbasFace(surf_bas->dof);
	    INT id; 
	    SHORT ibas_face[nbas_face];

        //FLOAT *normal = phgGeomGetFaceOutNormal(g, e, s);

        //if (!((e->bound_type[s] & BC_BOTTOM) && !(e->bound_type[s] & BC_ISHELF)))
        if (!(e->bound_type[s] & BC_BOTTOM))
            continue;

	    phgDofGetBasesOnFace(surf_dof, e, s, ibas_face);
	    //phgDofGetBasesOnFace(surf_bas->dof, e, s, ibas_face);
	    for (ii = 0; ii < nbas_face; ii++) {
		i = ibas_face[ii];
		dof_i = phgDofMapE2D(avg_n, e, i*Dim);
		avg_n_v = DofData(avg_n);
		normal[0] = avg_n_v[dof_i + 0];
		normal[1] = avg_n_v[dof_i + 1];
		normal[2] = avg_n_v[dof_i + 2];
        /* i means the base function number in the element */
		/* Use Gram–Schmidt process to get orthogonal bases, 
		 * one constrains */

		id = phgDofMapE2D(surf_dof, e, i * (DDim)) / (DDim);
		//id = phgDofMapE2D(surf_bas->dof, e, i * (DDim)) / (DDim);
		rotated[id] = TRUE;
		nrot++;
		
		BTYPE elem_btype = phgDofGetElementBoundaryType(surf_dof, e, i*DDim);
		//BTYPE elem_btype = phgDofGetElementBoundaryType(surf_bas->dof, e, i*DDim);


		/* fisrt basis */
		memcpy(H[0], normal, Dim * sizeof(FLOAT));

		/* second basis */
		for (m = 0; m < Dim; m++)
		    if (fabs(c[0] = INNER_PRODUCT(H[0], bxyz[m])) < 0.9)
			break;
		assert(m < Dim);
	
		H[1][0] = bxyz[m][0] - c[0] * H[0][0]; 
		H[1][1] = bxyz[m][1] - c[0] * H[0][1]; 
		H[1][2] = bxyz[m][2] - c[0] * H[0][2]; 
		norm = sqrt(INNER_PRODUCT(H[1], H[1]));
		assert(norm > 1e-10);
		H[1][0] /= norm;
		H[1][1] /= norm;
		H[1][2] /= norm;
		H[1][0] = 0;
		H[1][1] = 1;
		H[1][2] = 0;

		/* third basis */
		for (m++; m < Dim; m++)
		    if (fabs(c[0] = INNER_PRODUCT(H[0], bxyz[m])) < 0.9)
			break;
		assert(m < Dim);
		//c[1] = INNER_PRODUCT(H[1], bxyz[m]);
		c[1] = H[1][0]*(bxyz[m][0] - c[0] * H[0][0])+H[1][1]*(bxyz[m][1] - c[0] * H[0][1]) + H[1][2]*(bxyz[m][2] - c[0] * H[0][2]);
		H[2][0] = bxyz[m][0] - c[0] * H[0][0] - c[1] * H[1][0]; 
		H[2][1] = bxyz[m][1] - c[0] * H[0][1] - c[1] * H[1][1];  
		H[2][2] = bxyz[m][2] - c[0] * H[0][2] - c[1] * H[1][2];  
		H[2][0] = H[0][1]*H[1][2] - H[0][2]*H[1][1];
		H[2][1] = H[0][2]*H[1][0] - H[0][0]*H[1][2];
		H[2][2] = H[0][0]*H[1][1] - H[0][1]*H[1][0];
		norm = sqrt(INNER_PRODUCT(H[2], H[2]));
		assert(norm > 1e-10);
		H[2][0] /= norm;
		H[2][1] /= norm;
		H[2][2] /= norm;
        	
        if ((elem_btype & BC_BOTTOM_GRD) && (elem_btype & BC_DIVIDE))
        {
			
            H[1][0] = 1;
            H[1][1] = 0;
            H[1][2] = 0;

            H[2][0] = (H[0][1]*H[1][2]-H[0][2]*H[1][1]);
            H[2][1] = -(H[0][0]*H[1][2]-H[0][2]*H[1][0]);
            H[2][2] = (H[0][0]*H[1][1]-H[0][1]*H[1][0]);
            
        }


#if 0
#  warning check use only: xyz coord ----------------------------
		memcpy(bas_value[i], bxyz[0], DDim*sizeof(FLOAT));
#else
		memcpy(bas_value[i], H[0], DDim*sizeof(FLOAT));
        /* bas_value is contains all bas values of all nodes in the element. 
         * For the nodes at the boundaries, bas value is real number, otherwise
         * bas value is just 0 */
#endif
	    } /* end bas */
	}     /* end face */
	phgDofSetElementData(surf_dof, e, &bas_value[0][0]);
	//phgDofSetElementData(surf_bas->dof, e, &bas_value[0][0]);
    }	      /* end elem */
int
main(int argc, char *argv[])
{
    MAT *A, *B;
    VEC *U = NULL, *x;
    FLOAT *C;
    SOLVER *solver, *solver1 = NULL;
    INT i, j, k, n, *pvt, N = 1000, K = 2;
    char *main_opts = NULL, *sub_opts = NULL;

    phgOptionsRegisterInt("-n", "N value", &N);
    phgOptionsRegisterInt("-k", "K value", &K);
    phgOptionsRegisterString("-main_solver_opts", "Options for the main solver",
				&main_opts);
    phgOptionsRegisterString("-sub_solver_opts", "Options for the subsolver",
				&sub_opts);

    /* a direct solver is preferable for the sparse matrix */
    phgOptionsPreset("-solver mumps");
    phgInit(&argc, &argv);

    phgPrintf(
"----------------------------------------------------------------------------\n"
"This code solves (A+UU^t)x=b using the Sherman-Morrison-Woodbury formula.\n"
"Note: may use the following to disable use of the Sherman-Morrison-Woodbury\n"
"algorithm and change to the default solver instead:\n"
"	-preonly_pc_type solver -preonly_pc_opts \"-solver_maxit 2000\"\n"
"----------------------------------------------------------------------------\n"
    );

   phgPrintf("Generating the linear system: N = %"dFMT", K = %"dFMT"\n", N, K);

    /* A is a distributed NxN SPD tridiagonal matrix (A = [-1, 2, -1]) */
    n = N / phgNProcs + (phgRank < (N % phgNProcs) ? 1 : 0);
    A = phgMatCreate(phgComm, n, N);
    phgPrintf("  Generating matrix A.\n");
    for (i = 0; i < n; i++) {
	/* diagonal */
	phgMatAddEntry(A, i, i, 2.0);
	/* diagonal - 1 */
	if (i > 0)
	    phgMatAddEntry(A, i, i - 1, -1.0);
	else if (phgRank > 0)
	    phgMatAddLGEntry(A, i, A->rmap->partition[phgRank] - 1, -1.0);
	/* diagonal + 1 */
	if (i < n - 1)
	    phgMatAddEntry(A, i, i + 1, -1.0);
	else if (phgRank < phgNProcs - 1)
	    phgMatAddLGEntry(A, i, A->rmap->partition[phgRank] + n, -1.0);
    }
    phgMatAssemble(A);

    /* U is a K-component vector */
    U = phgMapCreateVec(A->rmap, K);
    phgVecRandomize(U, 123);

    /* solver1 is the solver for A */
    phgOptionsPush();
    phgOptionsSetOptions(sub_opts);
    solver1 = phgMat2Solver(SOLVER_DEFAULT, A);
    phgOptionsPop();

    /* x is a scratch vector */
    x = phgMapCreateVec(A->rmap, 1);

    /* C is a KxK dense matrix, pvt is an integer array, they store the LU
     * factorization of (I + U^t*inv(A)*U) */
    phgPrintf("  Generating the dense matrix I+U^t*inv(A)*U.\n");
    C = phgCalloc(K * K, sizeof(*C));
    pvt = phgAlloc(K * sizeof(*pvt));
    for (i = 0; i < K; i++) {
	for (j = 0; j < n; j++) {
	    solver1->rhs->data[j] = U->data[i * n + j];
	    x->data[j] = 0.0;
	}
	solver1->rhs->assembled = TRUE;
	phgSolverVecSolve(solver1, FALSE, x);
	for (j = 0; j < K; j++)
	    for (k = 0; k < n; k++)
		C[i * K + j] += U->data[j * n + k] * x->data[k];
    }
#if USE_MPI
    if (U->map->nprocs > 1) {
	FLOAT *tmp = phgAlloc(K * K * sizeof(*tmp));
	MPI_Allreduce(C, tmp, K * K, PHG_MPI_FLOAT, MPI_SUM, U->map->comm);
	phgFree(C);
	C = tmp;
    }
#endif	/* USE_MPI */
    for (i = 0; i < K; i++)
	C[i * K + i] += 1.0;
    phgPrintf("  Factorizing the dense matrix I+U^t*inv(A)*U.\n");
    phgSolverDenseLU(K, C, pvt);

    /* B is a matrix-free matrix representing A + U*U^t, B->mv_data is used
     * to pass A, U, solver1, C and pvt to callback functions */
    B = phgMapCreateMatrixFreeMat(A->rmap, A->cmap, funcB,
				  /* arguments carried over to CB functions */
				  A, U, solver1, C, pvt, NULL);

    /* solver is a PreOnly solver for B whose pc_proc is set to sherman().
     * 
     * Note: can also use pcg, gmres, or petsc for this solver, in this case
     * the solution obtained with the Sherman-Morisson formula is iteratively
     * refined. */
    phgOptionsPush();
    phgOptionsSetOptions("-solver preonly");
    phgOptionsSetOptions(main_opts);
    solver = phgMat2Solver(SOLVER_DEFAULT, B);
    phgSolverSetPC(solver, solver, sherman);
    phgOptionsPop();

    for (i = 0; i < n; i++)
	x->data[i] = 1.0;
    phgMatVec(MAT_OP_N, 1.0, B, x, 0.0, &solver->rhs);

    phgPrintf("Solving the linear system.\n");

    /* reset initial solution to zero */
    memset(x->data, 0, n * sizeof(*x->data));
    phgSolverVecSolve(solver, TRUE, x);
    for (i = 0; i < n; i++)
	solver->rhs->data[i] = 1.0;
    phgVecAXPBY(-1.0, solver->rhs, 1.0, &x);
    phgPrintf("Checking the result: |x - x_exact| / |x_exact| = %lg\n",
			(double)phgVecNorm2(x, 0, NULL) / sqrt((double)N));

    phgSolverDestroy(&solver);
    phgSolverDestroy(&solver1);
    phgMatDestroy(&A);
    phgMatDestroy(&B);
    phgVecDestroy(&U);
    phgVecDestroy(&x);
    phgFree(C);
    phgFree(pvt);

    phgFinalize();

    return 0;
}