PetscErrorCode MatSetUp_LMVM(Mat B)
{
Mat_LMVM *lmvm = (Mat_LMVM*)B->data;
PetscErrorCode ierr;
PetscInt m, n, M, N;
PetscMPIInt size;
MPI_Comm comm = PetscObjectComm((PetscObject)B);
PetscFunctionBegin;
ierr = MatGetSize(B, &M, &N);CHKERRQ(ierr);
if (M == 0 && N == 0) SETERRQ(comm, PETSC_ERR_ORDER, "MatSetSizes() must be called before MatSetUp()");
if (!lmvm->allocated) {
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
if (size == 1) {
ierr = VecCreateSeq(comm, N, &lmvm->Xprev);CHKERRQ(ierr);
ierr = VecCreateSeq(comm, M, &lmvm->Fprev);CHKERRQ(ierr);
} else {
ierr = MatGetLocalSize(B, &m, &n);CHKERRQ(ierr);
ierr = VecCreateMPI(comm, n, N, &lmvm->Xprev);CHKERRQ(ierr);
ierr = VecCreateMPI(comm, m, M, &lmvm->Fprev);CHKERRQ(ierr);
}
if (lmvm->m > 0) {
ierr = VecDuplicateVecs(lmvm->Xprev, lmvm->m, &lmvm->S);CHKERRQ(ierr);
ierr = VecDuplicateVecs(lmvm->Fprev, lmvm->m, &lmvm->Y);CHKERRQ(ierr);
}
lmvm->allocated = PETSC_TRUE;
B->preallocated = PETSC_TRUE;
B->assembled = PETSC_TRUE;
}
PetscFunctionReturn(0);
}
void TestPetscExceptions()
{
/*
* Note we could test with TS_ASSERT_THROWS_THIS() but PetscException
* includes line numbers so it isn't very robust.
*/
int err = 0;
TS_ASSERT_THROWS_NOTHING(PETSCEXCEPT(err));
Vec v;
err = VecCreateMPI(PETSC_COMM_WORLD, 2, 1, &v);
PetscTools::Destroy(v);
//#define PETSC_ERR_ARG_WRONGSTATE 73 /* object in argument is in wrong */
//#define PETSC_ERR_ARG_INCOMP 75 /* two arguments are incompatible */
TS_ASSERT_EQUALS(err, PETSC_ERR_ARG_INCOMP);
TS_ASSERT_THROWS(PETSCEXCEPT(err), Exception);
err=PETSC_ERR_FILE_OPEN;
//#define PETSC_ERR_FILE_OPEN 65 /* unable to open file */
TS_ASSERT_EQUALS(err, PETSC_ERR_FILE_OPEN);
TS_ASSERT_THROWS(PETSCEXCEPT(err), Exception);
// See if we can do it without a temporary
TS_ASSERT_THROWS(
PETSCEXCEPT(VecCreateMPI(PETSC_COMM_WORLD, 2, 1, &v)), Exception);
PetscTools::Destroy(v);
// This test give back an "unknown error" message
TS_ASSERT_THROWS( PETSCEXCEPT(-3), Exception);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx ctx;
TS ts;
Vec tsrhs,U;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetType(ts,TSEULER);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE,1,&tsrhs);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE,1,&U);CHKERRQ(ierr);
ierr = TSSetRHSFunction(ts,tsrhs,TSFunction,&ctx);CHKERRQ(ierr);
ctx.f = f;
ierr = SNESCreate(PETSC_COMM_WORLD,&ctx.snes);CHKERRQ(ierr);
ierr = SNESSetFromOptions(ctx.snes);CHKERRQ(ierr);
ierr = SNESSetFunction(ctx.snes,NULL,SNESFunction,&ctx);CHKERRQ(ierr);
ierr = SNESSetJacobian(ctx.snes,NULL,NULL,SNESComputeJacobianDefault,&ctx);CHKERRQ(ierr);
ctx.F = F;
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE,1,&ctx.V);CHKERRQ(ierr);
ierr = VecSet(U,1.0);CHKERRQ(ierr);
ierr = TSSolve(ts,U);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.V);CHKERRQ(ierr);
ierr = VecDestroy(&tsrhs);CHKERRQ(ierr);
ierr = VecDestroy(&U);CHKERRQ(ierr);
ierr = SNESDestroy(&ctx.snes);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void CCPoissonPETScLevelSolver::initializeSolverStateSpecialized(const SAMRAIVectorReal<NDIM, double>& x,
const SAMRAIVectorReal<NDIM, double>& /*b*/)
{
// Allocate DOF index data.
VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
const int x_idx = x.getComponentDescriptorIndex(0);
Pointer<CellDataFactory<NDIM, double> > x_fac = var_db->getPatchDescriptor()->getPatchDataFactory(x_idx);
const int depth = x_fac->getDefaultDepth();
Pointer<CellDataFactory<NDIM, int> > dof_index_fac =
var_db->getPatchDescriptor()->getPatchDataFactory(d_dof_index_idx);
dof_index_fac->setDefaultDepth(depth);
Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num);
if (!level->checkAllocated(d_dof_index_idx)) level->allocatePatchData(d_dof_index_idx);
// Setup PETSc objects.
int ierr;
PETScVecUtilities::constructPatchLevelDOFIndices(d_num_dofs_per_proc, d_dof_index_idx, level);
const int mpi_rank = SAMRAI_MPI::getRank();
ierr = VecCreateMPI(PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_x);
IBTK_CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_b);
IBTK_CHKERRQ(ierr);
PETScMatUtilities::constructPatchLevelCCLaplaceOp(
d_petsc_mat, d_poisson_spec, d_bc_coefs, d_solution_time, d_num_dofs_per_proc, d_dof_index_idx, level);
d_petsc_pc = d_petsc_mat;
d_petsc_ksp_ops_flag = SAME_PRECONDITIONER;
d_data_synch_sched = PETScVecUtilities::constructDataSynchSchedule(x_idx, level);
d_ghost_fill_sched = PETScVecUtilities::constructGhostFillSchedule(x_idx, level);
return;
} // initializeSolverStateSpecialized
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx ctx;
TS ts;
Vec tsrhs,U;
IS is;
PetscInt I;
PetscMPIInt rank;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetType(ts,TSEULER);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&tsrhs);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&U);CHKERRQ(ierr);
ierr = TSSetRHSFunction(ts,tsrhs,TSFunction,&ctx);CHKERRQ(ierr);
ctx.f = f;
ierr = SNESCreate(PETSC_COMM_WORLD,&ctx.snes);CHKERRQ(ierr);
ierr = SNESSetFromOptions(ctx.snes);CHKERRQ(ierr);
ierr = SNESSetFunction(ctx.snes,NULL,SNESFunction,&ctx);CHKERRQ(ierr);
ierr = SNESSetJacobian(ctx.snes,NULL,NULL,SNESComputeJacobianDefault,&ctx);CHKERRQ(ierr);
ctx.F = F;
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&ctx.V);CHKERRQ(ierr);
/* Create scatters to move between separate U and V representation and UV representation of solution */
ierr = VecCreateMPI(PETSC_COMM_WORLD,2,PETSC_DETERMINE,&ctx.UV);CHKERRQ(ierr);
I = 2*rank;
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,&I,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = VecScatterCreateWithData(U,NULL,ctx.UV,is,&ctx.scatterU);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
I = 2*rank + 1;
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,&I,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = VecScatterCreateWithData(ctx.V,NULL,ctx.UV,is,&ctx.scatterV);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecSet(U,1.0);CHKERRQ(ierr);
ierr = TSSolve(ts,U);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.V);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.UV);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx.scatterU);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx.scatterV);CHKERRQ(ierr);
ierr = VecDestroy(&tsrhs);CHKERRQ(ierr);
ierr = VecDestroy(&U);CHKERRQ(ierr);
ierr = SNESDestroy(&ctx.snes);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void StaggeredStokesPETScLevelSolver::initializeSolverStateSpecialized(
const SAMRAIVectorReal<NDIM, double>& x,
const SAMRAIVectorReal<NDIM, double>& /*b*/)
{
// Allocate DOF index data.
Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num);
if (!level->checkAllocated(d_u_dof_index_idx)) level->allocatePatchData(d_u_dof_index_idx);
if (!level->checkAllocated(d_p_dof_index_idx)) level->allocatePatchData(d_p_dof_index_idx);
// Setup PETSc objects.
int ierr;
StaggeredStokesPETScVecUtilities::constructPatchLevelDOFIndices(
d_num_dofs_per_proc, d_u_dof_index_idx, d_p_dof_index_idx, level);
const int mpi_rank = SAMRAI_MPI::getRank();
ierr = VecCreateMPI(
PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_x);
IBTK_CHKERRQ(ierr);
ierr = VecCreateMPI(
PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_b);
IBTK_CHKERRQ(ierr);
StaggeredStokesPETScMatUtilities::constructPatchLevelMACStokesOp(d_petsc_mat,
d_U_problem_coefs,
d_U_bc_coefs,
d_new_time,
d_num_dofs_per_proc,
d_u_dof_index_idx,
d_p_dof_index_idx,
level);
ierr = MatDuplicate(d_petsc_mat, MAT_COPY_VALUES, &d_petsc_pc);
IBTK_CHKERRQ(ierr);
HierarchyDataOpsManager<NDIM>* hier_ops_manager =
HierarchyDataOpsManager<NDIM>::getManager();
Pointer<HierarchyDataOpsInteger<NDIM> > hier_p_dof_index_ops =
hier_ops_manager->getOperationsInteger(d_p_dof_index_var, d_hierarchy, true);
hier_p_dof_index_ops->resetLevels(d_level_num, d_level_num);
const int min_p_idx = hier_p_dof_index_ops->min(
d_p_dof_index_idx); // NOTE: HierarchyDataOpsInteger::max() is broken
ierr = MatZeroRowsColumns(d_petsc_pc, 1, &min_p_idx, 1.0, NULL, NULL);
IBTK_CHKERRQ(ierr);
d_petsc_ksp_ops_flag = SAME_PRECONDITIONER;
const int u_idx = x.getComponentDescriptorIndex(0);
const int p_idx = x.getComponentDescriptorIndex(1);
d_data_synch_sched =
StaggeredStokesPETScVecUtilities::constructDataSynchSchedule(u_idx, p_idx, level);
d_ghost_fill_sched =
StaggeredStokesPETScVecUtilities::constructGhostFillSchedule(u_idx, p_idx, level);
return;
} // initializeSolverStateSpecialized
/*@
MatCreateNormalHermitian - Creates a new matrix object that behaves like (A*)'*A.
Collective on Mat
Input Parameter:
. A - the (possibly rectangular complex) matrix
Output Parameter:
. N - the matrix that represents (A*)'*A
Level: intermediate
Notes: The product (A*)'*A is NOT actually formed! Rather the new matrix
object performs the matrix-vector product by first multiplying by
A and then (A*)'
@*/
PetscErrorCode MatCreateNormalHermitian(Mat A,Mat *N)
{
PetscErrorCode ierr;
PetscInt m,n;
Mat_Normal *Na;
PetscFunctionBegin;
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),N);CHKERRQ(ierr);
ierr = MatSetSizes(*N,n,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscObjectChangeTypeName((PetscObject)*N,MATNORMALHERMITIAN);CHKERRQ(ierr);
ierr = PetscNewLog(*N,&Na);CHKERRQ(ierr);
(*N)->data = (void*) Na;
ierr = PetscObjectReference((PetscObject)A);CHKERRQ(ierr);
Na->A = A;
Na->scale = 1.0;
ierr = VecCreateMPI(PetscObjectComm((PetscObject)A),m,PETSC_DECIDE,&Na->w);CHKERRQ(ierr);
(*N)->ops->destroy = MatDestroyHermitian_Normal;
(*N)->ops->mult = MatMultHermitian_Normal;
(*N)->ops->multtranspose = MatMultHermitianTranspose_Normal;
(*N)->ops->multtransposeadd = MatMultHermitianTransposeAdd_Normal;
(*N)->ops->multadd = MatMultHermitianAdd_Normal;
(*N)->ops->getdiagonal = MatGetDiagonalHermitian_Normal;
(*N)->ops->scale = MatScaleHermitian_Normal;
(*N)->ops->diagonalscale = MatDiagonalScaleHermitian_Normal;
(*N)->assembled = PETSC_TRUE;
(*N)->cmap->N = A->cmap->N;
(*N)->rmap->N = A->cmap->N;
(*N)->cmap->n = A->cmap->n;
(*N)->rmap->n = A->cmap->n;
PetscFunctionReturn(0);
}
NonlocalCollection(Mat depends_on, IS interest_set) {
find_influences(depends_on, interest_set, &nodes);
PetscInt local_size;
ISGetLocalSize(nodes, &local_size);
VecCreateMPI(PETSC_COMM_WORLD, local_size, PETSC_DECIDE, &vec);
IS onto_index_set;
describe_partition(vec, &onto_index_set);
PetscInt begin;
PetscInt end;
VecGetOwnershipRange(vec, &begin, &end);
PetscInt *indicies;
ISGetIndices(nodes, &indicies);
assert(local_size == end-begin);
for (int ii=0; ii<local_size; ii++) {
map[indicies[ii]] = ii+begin;
}
ISRestoreIndices(nodes, &indicies);
Vec w;
MatGetVecs(depends_on, PETSC_NULL, &w);
VecScatterCreate(w, nodes, vec, onto_index_set, &scatter);
VecDestroy(w);
ISDestroy(onto_index_set);
}
/** Create a cache for Dirichlet part of closure vector, and scatter from global closure to Dirichlet cache.
@arg[in] gvec Global vector
@arg[out] dcache New vector to hold the Dirichlet values
@arg[out] dscat Scatter from global closure to \a dcache
@note This could be local but it doesn't cost anything to make it global.
**/
dErr VecDohpCreateDirichletCache(Vec gvec,Vec *dcache,VecScatter *dscat)
{
MPI_Comm comm;
dErr err;
dBool isdohp;
IS from;
Vec gc;
dInt n,nc,crstart;
dFunctionBegin;
dValidHeader(gvec,VEC_CLASSID,1);
dValidPointer(dcache,2);
dValidPointer(dscat,3);
err = PetscTypeCompare((PetscObject)gvec,VECDOHP,&isdohp);dCHK(err);
if (!isdohp) dERROR(PETSC_COMM_SELF,PETSC_ERR_SUP,"Vec type %s",((PetscObject)gvec)->type_name);
err = PetscObjectGetComm((PetscObject)gvec,&comm);dCHK(err);
err = VecGetLocalSize(gvec,&n);dCHK(err);
err = VecDohpGetClosure(gvec,&gc);dCHK(err);
err = VecGetLocalSize(gc,&nc);dCHK(err);
err = VecGetOwnershipRange(gc,&crstart,NULL);dCHK(err);
err = VecCreateMPI(comm,nc-n,PETSC_DECIDE,dcache);dCHK(err);
err = ISCreateStride(comm,nc-n,crstart+n,1,&from);dCHK(err);
err = VecScatterCreate(gc,from,*dcache,NULL,dscat);dCHK(err);
err = VecDohpRestoreClosure(gvec,&gc);dCHK(err);
err = ISDestroy(&from);dCHK(err);
/* \todo deal with rotations */
dFunctionReturn(0);
}
void FEMAssemble2DElasticity(MPI_Comm comm, Mesh *mesh, Mat &A, Vec &b,
PetscReal E, PetscReal mi, PetscReal(*dens)(Element *),
void(*f)(Element*, PetscReal, PetscReal*)) {
PetscInt rank;
MPI_Comm_rank(comm, &rank);
PetscInt size = mesh->vetrices.size();
MatCreateSeqAIJ(PETSC_COMM_SELF, size * 2, size * 2, 14, PETSC_NULL, &A);
VecCreateMPI(comm, size * 2, PETSC_DECIDE, &b);
for (std::map<PetscInt, Element*>::iterator e = mesh->elements.begin(); e
!= mesh->elements.end(); e++) {
Point* vetrices[3];
PetscInt vIndex[3];
for (int i = 0; i < 3; i++) {
vIndex[i] = e->second->vetrices[i];
vetrices[i] = mesh->vetrices[vIndex[i]];
}
PetscReal lStiff[36];
PetscReal bLoc[6];
//Local stiffness matrix and force vector assembly
PetscReal fs[2];
f(e->second, dens(e->second), fs);
elastLoc(vetrices, E, mi, fs, lStiff, bLoc);
PetscInt idx[6], idxLoc[6];
for (int i = 0; i < 3; i++) {
idx[i * 2] = vIndex[i] * 2;
idx[i * 2 + 1] = vIndex[i] * 2 + 1;
idxLoc[i * 2] = vIndex[i] * 2 - mesh->startIndexes[rank] * 2;
idxLoc[i * 2 + 1] = vIndex[i] * 2 + 1 - mesh->startIndexes[rank] * 2;
}
MatSetValues(A, 6, idxLoc, 6, idxLoc, lStiff, ADD_VALUES);
VecSetValues(b, 6, idx, bLoc, ADD_VALUES);
}
VecAssemblyBegin(b);
VecAssemblyEnd(b);
MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);
}
void
VCSCViscousPETScLevelSolver::initializeSolverStateSpecialized(const SAMRAIVectorReal<NDIM, double>& x,
const SAMRAIVectorReal<NDIM, double>& /*b*/)
{
// Allocate DOF index data.
VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
const int x_idx = x.getComponentDescriptorIndex(0);
Pointer<SideDataFactory<NDIM, double> > x_fac = var_db->getPatchDescriptor()->getPatchDataFactory(x_idx);
const int depth = x_fac->getDefaultDepth();
Pointer<SideDataFactory<NDIM, int> > dof_index_fac =
var_db->getPatchDescriptor()->getPatchDataFactory(d_dof_index_idx);
dof_index_fac->setDefaultDepth(depth);
if (!d_level->checkAllocated(d_dof_index_idx)) d_level->allocatePatchData(d_dof_index_idx);
PETScVecUtilities::constructPatchLevelDOFIndices(d_num_dofs_per_proc, d_dof_index_idx, d_level);
// Setup PETSc objects.
int ierr;
const int mpi_rank = SAMRAI_MPI::getRank();
ierr = VecCreateMPI(PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_x);
IBTK_CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_b);
IBTK_CHKERRQ(ierr);
const double alpha = 1.0;
const double beta = 1.0;
PETScMatUtilities::constructPatchLevelVCSCViscousOp(d_petsc_mat,
d_poisson_spec,
alpha,
beta,
d_bc_coefs,
d_solution_time,
d_num_dofs_per_proc,
d_dof_index_idx,
d_level,
d_mu_interp_type);
d_petsc_pc = d_petsc_mat;
// Setup SAMRAI communication objects.
d_data_synch_sched = PETScVecUtilities::constructDataSynchSchedule(x_idx, d_level);
d_ghost_fill_sched = PETScVecUtilities::constructGhostFillSchedule(x_idx, d_level);
return;
} // initializeSolverStateSpecialized
PetscErrorCode FEMAssembleTotal2DLaplace(MPI_Comm comm, Mesh *mesh, Mat &A,
Vec &b, PetscReal(*f)(Point), PetscReal(*K)(Point)) {
PetscErrorCode ierr;
PetscInt rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
PetscInt size = mesh->vetrices.size();
ierr
= MatCreateMPIAIJ(comm, size, size, PETSC_DECIDE, PETSC_DECIDE, 7, PETSC_NULL, 0, PETSC_NULL, &A);
CHKERRQ(ierr);
ierr = VecCreateMPI(comm, size, PETSC_DECIDE, &b);
CHKERRQ(ierr);
for (std::map<PetscInt, Element*>::iterator e = mesh->elements.begin(); e
!= mesh->elements.end(); e++) {
PetscScalar bl[3];
PetscScalar Al[9];
PetscReal R[4];
PetscInt elSize = e->second->numVetrices;
Point *vetrices = new Point[elSize];
PetscInt ixs[elSize];
for (int j = 0; j < elSize; j++) {
ixs[j] = e->second->vetrices[j];
vetrices[j] = *(mesh->vetrices[ixs[j]]);
}
R[0] = vetrices[1].x - vetrices[0].x;
R[2] = vetrices[1].y - vetrices[0].y;
R[1] = vetrices[2].x - vetrices[0].x;
R[3] = vetrices[2].y - vetrices[0].y;
Point center = getCenterOfSet(vetrices, elSize);
bLoc(R, bl, f(center));
ALoc(R, Al, K(center));
ierr = VecSetValues(b, elSize, ixs, bl, ADD_VALUES);
CHKERRQ(ierr);
ierr = MatSetValues(A, elSize, ixs, elSize, ixs, Al, ADD_VALUES);
CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(b);
CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
return ierr;
}
PkStruct::PkStruct(double _kmin, double _dk, int _Nbins) {
kmin = _kmin; dk = _dk; Nbins = _Nbins;
VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, Nbins, &kvec);
VecDuplicate(kvec, &pkvec);
VecDuplicate(kvec, &nmodes);
VecSet(kvec,0.0);
VecSet(pkvec, 0.0);
VecSet(nmodes, 0.0);
VecGetOwnershipRange(kvec, &lo, &hi);
}
/*@
PFApplyVec - Applies the mathematical function to a vector
Collective on PF
Input Parameters:
+ pf - the function context
- x - input vector (or PETSC_NULL for the vector (0,1, .... N-1)
Output Parameter:
. y - output vector
Level: beginner
.keywords: PF, apply
.seealso: PFApply(), PFCreate(), PFDestroy(), PFSetType(), PFSet()
@*/
PetscErrorCode PFApplyVec(PF pf,Vec x,Vec y)
{
PetscErrorCode ierr;
PetscInt i,rstart,rend,n,p;
PetscBool nox = PETSC_FALSE;
PetscFunctionBegin;
PetscValidHeaderSpecific(pf,PF_CLASSID,1);
PetscValidHeaderSpecific(y,VEC_CLASSID,3);
if (x) {
PetscValidHeaderSpecific(x,VEC_CLASSID,2);
if (x == y) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_IDN,"x and y must be different vectors");
} else {
PetscScalar *xx;
PetscInt lsize;
ierr = VecGetLocalSize(y,&lsize);CHKERRQ(ierr);
lsize = pf->dimin*lsize/pf->dimout;
ierr = VecCreateMPI(((PetscObject)y)->comm,lsize,PETSC_DETERMINE,&x);CHKERRQ(ierr);
nox = PETSC_TRUE;
ierr = VecGetOwnershipRange(x,&rstart,&rend);CHKERRQ(ierr);
ierr = VecGetArray(x,&xx);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
xx[i-rstart] = (PetscScalar)i;
}
ierr = VecRestoreArray(x,&xx);CHKERRQ(ierr);
}
ierr = VecGetLocalSize(x,&n);CHKERRQ(ierr);
ierr = VecGetLocalSize(y,&p);CHKERRQ(ierr);
if ((pf->dimin*(n/pf->dimin)) != n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Local input vector length %D not divisible by dimin %D of function",n,pf->dimin);
if ((pf->dimout*(p/pf->dimout)) != p) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Local output vector length %D not divisible by dimout %D of function",p,pf->dimout);
if ((n/pf->dimin) != (p/pf->dimout)) SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Local vector lengths %D %D are wrong for dimin and dimout %D %D of function",n,p,pf->dimin,pf->dimout);
if (pf->ops->applyvec) {
ierr = (*pf->ops->applyvec)(pf->data,x,y);CHKERRQ(ierr);
} else {
PetscScalar *xx,*yy;
ierr = VecGetLocalSize(x,&n);CHKERRQ(ierr);
n = n/pf->dimin;
ierr = VecGetArray(x,&xx);CHKERRQ(ierr);
ierr = VecGetArray(y,&yy);CHKERRQ(ierr);
if (!pf->ops->apply) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"No function has been provided for this PF");
ierr = (*pf->ops->apply)(pf->data,n,xx,yy);CHKERRQ(ierr);
ierr = VecRestoreArray(x,&xx);CHKERRQ(ierr);
ierr = VecRestoreArray(y,&yy);CHKERRQ(ierr);
}
if (nox) {
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PETSC_INTERN PetscErrorCode MatConvert_Shell(Mat oldmat, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat mat;
Vec in,out;
PetscErrorCode ierr;
PetscInt i,M,m,*rows,start,end;
MPI_Comm comm;
PetscScalar *array,zero = 0.0,one = 1.0;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)oldmat,&comm);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(oldmat,&start,&end);CHKERRQ(ierr);
ierr = VecCreateMPI(comm,end-start,PETSC_DECIDE,&in);CHKERRQ(ierr);
ierr = VecDuplicate(in,&out);CHKERRQ(ierr);
ierr = VecGetSize(in,&M);CHKERRQ(ierr);
ierr = VecGetLocalSize(in,&m);CHKERRQ(ierr);
ierr = PetscMalloc1(m+1,&rows);CHKERRQ(ierr);
for (i=0; i<m; i++) rows[i] = start + i;
ierr = MatCreate(comm,&mat);CHKERRQ(ierr);
ierr = MatSetSizes(mat,m,M,M,M);CHKERRQ(ierr);
ierr = MatSetType(mat,newtype);CHKERRQ(ierr);
ierr = MatSetBlockSizesFromMats(mat,oldmat,oldmat);CHKERRQ(ierr);
ierr = MatSetUp(mat);CHKERRQ(ierr);
for (i=0; i<M; i++) {
ierr = VecSet(in,zero);CHKERRQ(ierr);
ierr = VecSetValues(in,1,&i,&one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(in);CHKERRQ(ierr);
ierr = VecAssemblyEnd(in);CHKERRQ(ierr);
ierr = MatMult(oldmat,in,out);CHKERRQ(ierr);
ierr = VecGetArray(out,&array);CHKERRQ(ierr);
ierr = MatSetValues(mat,m,rows,1,&i,array,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecRestoreArray(out,&array);CHKERRQ(ierr);
}
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = VecDestroy(&in);CHKERRQ(ierr);
ierr = VecDestroy(&out);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_INPLACE_MATRIX) {
ierr = MatHeaderReplace(oldmat,&mat);CHKERRQ(ierr);
} else {
*newmat = mat;
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
Vec x,y;
IS is1,is2;
PetscInt n,N,ix[2],iy[2];
VecScatter ctx;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size < 3) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_OUTOFRANGE,"This example needs at least 3 processes");
/* create two vectors */
n = 2;
N = 2*size;
ierr = VecCreateMPI(PETSC_COMM_WORLD,n,N,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
/* Specify indices to send from the next process in the ring */
ix[0] = ((rank+1)*n+0) % N;
ix[1] = ((rank+1)*n+1) % N;
/* And put them on the process after that in the ring */
iy[0] = ((rank+2)*n+0) % N;
iy[1] = ((rank+2)*n+1) % N;
/* create two index sets */
ierr = ISCreateGeneral(PETSC_COMM_WORLD,n,ix,PETSC_USE_POINTER,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,n,iy,PETSC_USE_POINTER,&is2);CHKERRQ(ierr);
ierr = VecSetValue(x,rank*n,rank*n,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValue(x,rank*n+1,rank*n+1,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"----\n");CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
DMCreateGlobalVector_SNESVI - Creates global vector of the size of the reduced space
*/
PetscErrorCode DMCreateGlobalVector_SNESVI(DM dm,Vec *vec)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)dm,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_PLIB,"Composed SNES is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi);CHKERRQ(ierr);
ierr = VecCreateMPI(((PetscObject)dm)->comm,dmsnesvi->n,PETSC_DETERMINE,vec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETScVector::PETScVector(const PetscInt vec_size, const bool is_global_size)
{
if( is_global_size ) {
VecCreate(PETSC_COMM_WORLD, &_v);
VecSetSizes(_v, PETSC_DECIDE, vec_size);
} else {
// Fix size partitioning
// the size can be associated to specific memory allocation of a matrix
VecCreateMPI(PETSC_COMM_WORLD, vec_size, PETSC_DECIDE, &_v);
}
config();
}
/*@C
MatCreateSubMatrixFree - Creates a reduced matrix by masking a
full matrix.
Collective on matrix
Input Parameters:
+ mat - matrix of arbitrary type
. Rows - the rows that will be in the submatrix
- Cols - the columns that will be in the submatrix
Output Parameters:
. J - New matrix
Notes:
The user provides the input data and is responsible for destroying
this data after matrix J has been destroyed.
Level: developer
.seealso: MatCreate()
@*/
PetscErrorCode MatCreateSubMatrixFree(Mat mat,IS Rows, IS Cols, Mat *J)
{
MPI_Comm comm=PetscObjectComm((PetscObject)mat);
MatSubMatFreeCtx ctx;
PetscErrorCode ierr;
PetscMPIInt size;
PetscInt mloc,nloc,m,n;
PetscFunctionBegin;
ierr = PetscNew(&ctx);CHKERRQ(ierr);
ctx->A=mat;
ierr = MatGetSize(mat,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(mat,&mloc,&nloc);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) {
ierr = VecCreateSeq(comm,n,&ctx->VC);CHKERRQ(ierr);
} else {
ierr = VecCreateMPI(comm,nloc,n,&ctx->VC);CHKERRQ(ierr);
}
ctx->VR=ctx->VC;
ierr = PetscObjectReference((PetscObject)mat);CHKERRQ(ierr);
ctx->Rows = Rows;
ctx->Cols = Cols;
ierr = PetscObjectReference((PetscObject)Rows);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)Cols);CHKERRQ(ierr);
ierr = MatCreateShell(comm,mloc,nloc,m,n,ctx,J);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_MULT,(void(*)(void))MatMult_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_DESTROY,(void(*)(void))MatDestroy_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_VIEW,(void(*)(void))MatView_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_MULT_TRANSPOSE,(void(*)(void))MatMultTranspose_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_DIAGONAL_SET,(void(*)(void))MatDiagonalSet_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_SHIFT,(void(*)(void))MatShift_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_EQUAL,(void(*)(void))MatEqual_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_SCALE,(void(*)(void))MatScale_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_TRANSPOSE,(void(*)(void))MatTranspose_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_GET_DIAGONAL,(void(*)(void))MatGetDiagonal_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_GET_SUBMATRICES,(void(*)(void))MatGetSubMatrices_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_NORM,(void(*)(void))MatNorm_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_DUPLICATE,(void(*)(void))MatDuplicate_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_GET_SUBMATRIX,(void(*)(void))MatGetSubMatrix_SMF);CHKERRQ(ierr);
ierr = MatShellSetOperation(*J,MATOP_GET_ROW_MAX,(void(*)(void))MatDuplicate_SMF);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)mat,(PetscObject)(*J));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
EXTERN_C_END
#undef __FUNCT__
#define __FUNCT__ "DMCreateGlobalVector_Composite"
PetscErrorCode DMCreateGlobalVector_Composite(DM dm,Vec *gvec)
{
PetscErrorCode ierr;
DM_Composite *com = (DM_Composite*)dm->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = VecCreateMPI(((PetscObject)dm)->comm,com->n,com->N,gvec);CHKERRQ(ierr);
ierr = VecSetDM(*gvec, dm);CHKERRQ(ierr);
ierr = VecSetOperation(*gvec,VECOP_VIEW,(void(*)(void))VecView_DMComposite);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx ctx;
TS ts;
Vec tsrhs,UV;
IS is;
PetscInt I;
PetscMPIInt rank;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetType(ts,TSROSW);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,2,PETSC_DETERMINE,&tsrhs);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,2,PETSC_DETERMINE,&UV);CHKERRQ(ierr);
ierr = TSSetRHSFunction(ts,tsrhs,TSFunctionRHS,&ctx);CHKERRQ(ierr);
ierr = TSSetIFunction(ts,NULL,TSFunctionI,&ctx);CHKERRQ(ierr);
ctx.f = f;
ctx.F = F;
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&ctx.U);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&ctx.V);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&ctx.UF);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,1,PETSC_DETERMINE,&ctx.VF);CHKERRQ(ierr);
I = 2*rank;
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,&I,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = VecScatterCreate(ctx.U,NULL,UV,is,&ctx.scatterU);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
I = 2*rank + 1;
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,&I,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = VecScatterCreate(ctx.V,NULL,UV,is,&ctx.scatterV);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecSet(UV,1.0);CHKERRQ(ierr);
ierr = TSSolve(ts,UV);CHKERRQ(ierr);
ierr = VecDestroy(&tsrhs);CHKERRQ(ierr);
ierr = VecDestroy(&UV);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.U);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.V);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.UF);CHKERRQ(ierr);
ierr = VecDestroy(&ctx.VF);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx.scatterU);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx.scatterV);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
/*
Rozdeleni site mezi procesory takto:
1) vrcholy jsou nakopirovany na vsechny procesory
2) bunky jsou rozdeleny rovnmerne dle petsc
3) steny jsou rozdeleny tak, ze cpu(f) = cpu(cell(f.owner))
Indexy bunek u sten jsou zmeneny na lokalni cislovani
*/
Grid DecomposeGrid(const Grid& g) {
PetscInt istart, iend;
Vec tmp;
VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, g.cells().size(), &tmp);
VecGetOwnershipRange(tmp, &istart, &iend);
VecDestroy(&tmp);
Grid::CellList cells(0);
for (auto c: g.cells())
if (istart <= c.id && c.id < iend) cells.push_back(c);
Grid::FaceList faces;
for (auto f: g.internalFaces())
if (istart <= f.owner && f.owner < iend) faces.push_back(f);
for (auto bnd: g.boundaryPatches())
for (auto f: bnd.second)
if (istart <= f.owner && f.owner < iend) faces.push_back(f);
Decomposition dist(istart, iend, faces);
for (auto& f: faces) {
int glob[1], loc[1], one;
glob[0] = f.owner;
ISGlobalToLocalMappingApply(dist.locToGlobMap, IS_GTOLM_MASK, 1, glob, &one, loc);
f.owner = loc[0];
if (f.neighbour >=0) {
glob[0] = f.neighbour;
ISGlobalToLocalMappingApply(dist.locToGlobMap, IS_GTOLM_MASK, 1, glob, &one, loc);
f.neighbour = loc[0];
}
}
Grid dg(g.points(), faces, cells);
dg.userData() = dist;
return dg;
}
int main(int argc,char **args)
{
Vec b;
PetscViewer fd;
PetscErrorCode ierr;
PetscInt i;
PetscInitialize(&argc,&args,(char *)0,help);
/* server indicates we WAIT for someone to connect to our socket */
ierr = PetscViewerSocketOpen(PETSC_COMM_WORLD,"server",PETSC_DEFAULT,&fd);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,10000,PETSC_DECIDE,&b);CHKERRQ(ierr);
for (i=0;i<1000;i++){
ierr = VecView(b,fd);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecLoad(b,fd);CHKERRQ(ierr);
}
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
inline void PetscVector::init (const int n,
const int n_local,
const bool fast,
const ParallelType type) {
int ierr=0;
int petsc_n=static_cast<int>(n);
int petsc_n_local=static_cast<int>(n_local);
// Clear initialized vectors
if (this->initialized()) this->clear();
if (type == AUTOMATIC) {
if (n == n_local) this->_type = SERIAL;
else this->_type = PARALLEL;
} else this->_type = type;
assert ((this->_type==SERIAL && n==n_local) || this->_type==PARALLEL);
// create a sequential vector if on only 1 processor
if (this->_type == SERIAL) {
ierr = VecCreateSeq (PETSC_COMM_SELF, petsc_n, &_vec);
CHKERRABORT(PETSC_COMM_SELF,ierr);
ierr = VecSetFromOptions (_vec);
CHKERRABORT(PETSC_COMM_SELF,ierr);
}
// otherwise create an MPI-enabled vector
else if (this->_type == PARALLEL) {
assert (n_local <= n);
ierr = VecCreateMPI (MPI_COMM_WORLD, petsc_n_local, petsc_n, &_vec);
CHKERRABORT(MPI_COMM_WORLD,ierr);
ierr = VecSetFromOptions (_vec);
CHKERRABORT(MPI_COMM_WORLD,ierr);
} else {
std::cout << "Not good" <<std::endl;
abort();
}
this->_is_initialized = true;
this->_is_closed = true;
if (fast == false) this->zero ();
}
PetscErrorCode ConvertVectorToVec(MPI_Comm comm,vector *v,Vec *Pv)
{
PetscErrorCode ierr;
PetscMPIInt size,rank;
int m,M,i,*mnls,*start_indices,*global_indices;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
m = v->mnl;
M = v->n;
ierr = VecCreateMPI(comm,m,M,Pv);CHKERRQ(ierr);
ierr = PetscMalloc(size*sizeof(int),&mnls);CHKERRQ(ierr);
ierr = MPI_Allgather(&v->mnl,1,MPI_INT,mnls,1,MPI_INT,comm);CHKERRQ(ierr);
ierr = PetscMalloc(size*sizeof(int),&start_indices);CHKERRQ(ierr);
start_indices[0] = 0;
for (i=1; i<size; i++)
start_indices[i] = start_indices[i-1] +mnls[i-1];
ierr = PetscMalloc(v->mnl*sizeof(int),&global_indices);CHKERRQ(ierr);
for (i=0; i<v->mnl; i++)
global_indices[i] = start_indices[rank] + i;
ierr = PetscFree(mnls);CHKERRQ(ierr);
ierr = PetscFree(start_indices);CHKERRQ(ierr);
ierr = VecSetValues(*Pv,v->mnl,global_indices,v->v,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(*Pv);CHKERRQ(ierr);
ierr = VecAssemblyEnd(*Pv);CHKERRQ(ierr);
ierr = PetscFree(global_indices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
MSA_BoundaryConditions - Calculates the boundary conditions for
the region.
Input Parameter:
. user - user-defined application context
Output Parameter:
. user - user-defined application context
*/
PetscErrorCode MSA_BoundaryConditions(AppCtx * user)
{
PetscErrorCode ierr;
PetscInt i,j,k,limit=0,maxits=5;
PetscInt mx=user->mx,my=user->my;
PetscInt xs,ys,xm,ym;
PetscInt bsize=0, lsize=0, tsize=0, rsize=0;
PetscScalar one =1.0, two=2.0, three=3.0, tol=1e-10;
PetscScalar fnorm,det,hx,hy,xt=0,yt=0;
PetscScalar u1,u2,nf1,nf2,njac11,njac12,njac21,njac22;
PetscScalar b=-0.5, t=0.5, l=-0.5, r=0.5;
PetscScalar *boundary;
Vec Bottom,Top,Right,Left;
PetscScalar scl=1.0;
PetscBool flg;
PetscFunctionBeginUser;
ierr = DMDAGetCorners(user->da,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
bsize=xm+2; lsize=ym+2; rsize=ym+2; tsize=xm+2;
ierr = VecCreateMPI(PETSC_COMM_WORLD,bsize,PETSC_DECIDE,&Bottom);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,tsize,PETSC_DECIDE,&Top);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,lsize,PETSC_DECIDE,&Left);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,rsize,PETSC_DECIDE,&Right);CHKERRQ(ierr);
user->Top = Top;
user->Left = Left;
user->Bottom = Bottom;
user->Right = Right;
hx= (r-l)/(mx+1); hy=(t-b)/(my+1);
for (j=0; j<4; j++) {
if (j==0) {
yt = b;
xt = l+hx*xs;
limit = bsize;
ierr = VecGetArray(Bottom,&boundary);CHKERRQ(ierr);
} else if (j==1) {
yt = t;
xt = l+hx*xs;
limit= tsize;
ierr = VecGetArray(Top,&boundary);CHKERRQ(ierr);
} else if (j==2) {
yt = b+hy*ys;
xt = l;
limit= lsize;
ierr = VecGetArray(Left,&boundary);CHKERRQ(ierr);
} else { /* if (j==3) */
yt = b+hy*ys;
xt = r;
limit= rsize;
ierr = VecGetArray(Right,&boundary);CHKERRQ(ierr);
}
for (i=0; i<limit; i++) {
u1=xt;
u2=-yt;
for (k=0; k<maxits; k++) {
nf1 = u1 + u1*u2*u2 - u1*u1*u1/three-xt;
nf2 = -u2 - u1*u1*u2 + u2*u2*u2/three-yt;
fnorm = PetscSqrtReal(nf1*nf1+nf2*nf2);
if (fnorm <= tol) break;
njac11 = one+u2*u2-u1*u1;
njac12 = two*u1*u2;
njac21 = -two*u1*u2;
njac22 = -one - u1*u1 + u2*u2;
det = njac11*njac22-njac21*njac12;
u1 = u1-(njac22*nf1-njac12*nf2)/det;
u2 = u2-(njac11*nf2-njac21*nf1)/det;
}
boundary[i]=u1*u1-u2*u2;
if (j==0 || j==1) xt=xt+hx;
else yt=yt+hy; /* if (j==2 || j==3) */
}
if (j==0) {
ierr = VecRestoreArray(Bottom,&boundary);CHKERRQ(ierr);
} else if (j==1) {
ierr = VecRestoreArray(Top,&boundary);CHKERRQ(ierr);
} else if (j==2) {
ierr = VecRestoreArray(Left,&boundary);CHKERRQ(ierr);
} else if (j==3) {
ierr = VecRestoreArray(Right,&boundary);CHKERRQ(ierr);
}
}
/* Scale the boundary if desired */
ierr = PetscOptionsGetReal(NULL,"-bottom",&scl,&flg);CHKERRQ(ierr);
if (flg) {
ierr = VecScale(Bottom, scl);CHKERRQ(ierr);
}
ierr = PetscOptionsGetReal(NULL,"-top",&scl,&flg);CHKERRQ(ierr);
if (flg) {
ierr = VecScale(Top, scl);CHKERRQ(ierr);
}
ierr = PetscOptionsGetReal(NULL,"-right",&scl,&flg);CHKERRQ(ierr);
if (flg) {
ierr = VecScale(Right, scl);CHKERRQ(ierr);
}
ierr = PetscOptionsGetReal(NULL,"-left",&scl,&flg);CHKERRQ(ierr);
if (flg) {
ierr = VecScale(Left, scl);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode BearQueryMat(PetscInt s, PetscScalar c, Mat invL1, Mat invU1, Mat invL2, Mat invU2, Mat H12, Mat H21, Vec order){
PetscErrorCode err;
PetscInt n1, n2, n, M, N;
PetscInt oseed;
PetscScalar val, one = 1.0;
PetscMPIInt size;
PetscLogDouble tic, toc;
Mat r = NULL;
Mat r1 = NULL, q1 = NULL, t1_1 = NULL, t1_2 = NULL, t1_3 = NULL, t1_4 = NULL, t1_5 = NULL; // dimension: n1
Mat r2 = NULL, q2 = NULL, q_tilda = NULL, t2_1 = NULL, t2_2 = NULL, t2_3 = NULL; // dimension: n2_idx
Vec vr=NULL, vr1=NULL, vr2=NULL;
PetscInt col = 0;
err = MPI_Comm_size(PETSC_COMM_WORLD, &size); CHKERRQ(err);
err = MatGetSize(H12, &n1, &n2); CHKERRQ(err);
n = n1 + n2;
err = PetscPrintf(PETSC_COMM_WORLD, "n1: %d, n2: %d\n", n1, n2); CHKERRQ(err);
err = MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, 1, n, size, 1, NULL, 1, NULL, &r); CHKERRQ(err);
err = MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, 1, n1, size, 1, NULL, 1, NULL, &q1); CHKERRQ(err);
err = MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, 1, n2, size, 1, NULL, 1, NULL, &q2); CHKERRQ(err);
// err = MatCreate(PETSC_COMM_WORLD, &q2); CHKERRQ(err);
// err = MatSetSizes(q2, PETSC_DECIDE, PETSC_DECIDE, n2, 1); CHKERRQ(err);
// err = MatSetType(q2, MATAIJ); CHKERRQ(err);
// err = MatSetUp(q2);
s = s - 1; // shift -1 for zero-based index
err = VecGetValues(order, 1, &s, &val); CHKERRQ(err);
oseed = (PetscInt) val;
// err = PetscPrintf(PETSC_COMM_WORLD, "Given seed: %d, Reorered seed: %d (0 ~ n-1)\n", s, oseed); CHKERRQ(err);
if(oseed < n1){
//err = MatSetValues(q1, 1, &oseed, 1, &col, &one, INSERT_VALUES); CHKERRQ(err);
err = MatSetValue(q1, oseed, col, one, INSERT_VALUES); CHKERRQ(err);
}else{
oseed = oseed - n1;
//err = MatSetValues(q2, 1, &oseed, 1, &col, &one, INSERT_VALUES); CHKERRQ(err);
err = MatSetValue(q2, oseed, col, one, INSERT_VALUES); CHKERRQ(err);
//err = printVecSum(q2);
}
err = MatAssemblyBegin(q1, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = MatAssemblyEnd(q1, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = MatAssemblyBegin(q2, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = MatAssemblyEnd(q2, MAT_FINAL_ASSEMBLY); CHKERRQ(err);
err = printMatInfo("q1", q1);
err = printMatInfo("q2", q2);
//err = MatView(q1, PETSC_VIEWER_STDOUT_WORLD);
//err = MatView(q2, PETSC_VIEWER_STDOUT_WORLD);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &r1); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_1); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_2); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_3); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_4); CHKERRQ(err);
err = MatDuplicate(q1, MAT_DO_NOT_COPY_VALUES, &t1_5); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &r2); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &q_tilda); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &t2_1); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &t2_2); CHKERRQ(err);
err = MatDuplicate(q2, MAT_DO_NOT_COPY_VALUES, &t2_3); CHKERRQ(err);
err = MatMatMult(invL1, q1, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_1); CHKERRQ(err);
err = MatMatMult(invU1, t1_1, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_2); CHKERRQ(err);
err = MatMatMult(H21, t1_2, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t2_1); CHKERRQ(err);
err = MatScale(t2_1, -1.0); CHKERRQ(err);
err = MatAXPY(t2_1, 1.0, q2, DIFFERENT_NONZERO_PATTERN); CHKERRQ(err);
//MatView(t1_1, PETSC_VIEWER_STDOUT_WORLD);
err = PetscTime(&tic);
err = MatMatMult(invL2, t2_1, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t2_2); CHKERRQ(err);
err = MatMatMult(invU2, t2_2, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &r2); CHKERRQ(err);
err = PetscTime(&toc);
err = PetscPrintf(PETSC_COMM_WORLD, "running time: %f sec\n", toc-tic); CHKERRQ(err);
err = MatMatMult(H12, r2, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_3); CHKERRQ(err);
err = MatScale(t1_3, -1.0); CHKERRQ(err);
err = MatAXPY(t1_3, 1.0, q1, DIFFERENT_NONZERO_PATTERN); CHKERRQ(err);
err = MatMatMult(invL1, t1_3, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &t1_5); CHKERRQ(err);
err = MatMatMult(invU1, t1_5, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &r1); CHKERRQ(err);
//MatView(r1, PETSC_VIEWER_STDOUT_WORLD);
MatGetSize(r1, &M, &N);
PetscPrintf(PETSC_COMM_WORLD, "%d %d\n", M, N);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n1, &vr1);
err = MatGetColumnVector(r1, vr1, 0);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n2, &vr2);
err = MatGetColumnVector(r2, vr2, 0);
err = printMatInfo("r2", r2);
/*
// Start matrix-vec multiplications
err = MatMult(invU2, t2_2, r2); CHKERRQ(err);
err = MatMult(H12, r2, t1_3); CHKERRQ(err);
err = VecAXPBYPCZ(t1_4, 1.0, -1.0, 0.0, q1, t1_3); CHKERRQ(err);
err = MatMult(invL1, t1_4, t1_5); CHKERRQ(err);
err = MatMult(invU1, t1_5, r1); CHKERRQ(err);
//err = printVecSum(r1);
//err = VecView(r2, PETSC_VIEWER_STDOUT_WORLD);
// Concatenate r1 and r2
err = VecMerge(r1, r2, r); CHKERRQ(err);
err = VecScale(r, c); CHKERRQ(err);
//err = VecView(r, PETSC_VIEWER_STDOUT_WORLD);
//err = VecDuplicate(r, &or); CHKERRQ(err);
err = VecReorder(r, order, or); CHKERRQ(err);
//err = VecView(or, PETSC_VIEWER_STDOUT_WORLD);
*/
err = MatDestroy(&r); CHKERRQ(err);
err = MatDestroy(&r1); CHKERRQ(err);
err = MatDestroy(&q1); CHKERRQ(err);
err = MatDestroy(&t1_1); CHKERRQ(err);
err = MatDestroy(&t1_2); CHKERRQ(err);
err = MatDestroy(&t1_3); CHKERRQ(err);
err = MatDestroy(&t1_4); CHKERRQ(err);
err = MatDestroy(&t1_5); CHKERRQ(err);
err = MatDestroy(&r2); CHKERRQ(err);
err = MatDestroy(&q2); CHKERRQ(err);
err = MatDestroy(&q_tilda); CHKERRQ(err);
err = MatDestroy(&t2_1); CHKERRQ(err);
err = MatDestroy(&t2_2); CHKERRQ(err);
err = MatDestroy(&t2_3); CHKERRQ(err);
return err;
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscErrorCode ierr;
Vec x,y0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,l,n,p,m,m2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar dx,dy,dx2,dy2;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput;
char fname[50];
PetscViewer socketviewer;
PetscInt withMatlab;
PetscTruth Matlabflag;
PetscLogDouble v1,v2,elapsed_time;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
dx2 = dx/2-dx/1e6;
dy2 = dy/2-dy/1e6;
npt = 5;
pmax = 4e6;
puse = pmax;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*16 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*2);
/////////////////////////////////////////////////////////////////////////////////////
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
// Create initial vector
Vec x0;
PetscScalar *x0array;
x0array = malloc((localsize)*n*sizeof(PetscScalar));
k = 0;
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
*(x0array+k) = cos(2*M_PI*(dx/2+i*dx));
//*(x0array+k) = cos(2*M_PI*(dy/2+j*dy));
k++;
}
}
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,n*localsize,PETSC_DECIDE,x0array,&x0);CHKERRQ(ierr);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pmax*npt,&y0);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
///////////////////////////////////////////////////////////////////////////
// Map Center Points
PetscInt *NzindJ,*idx,*idy,*idp;
PetscScalar *CenterX,*CenterY,*VecVal,*pty;
PetscScalar *ShiftX,*ShiftY,CX,CY, *yarray;
IS isx,isy;
VecScatter ctx;
CenterX = malloc(npt*sizeof(PetscScalar));
CenterY = malloc(npt*sizeof(PetscScalar));
ShiftX = malloc(npt*sizeof(PetscScalar));
ShiftY = malloc(npt*sizeof(PetscScalar));
VecVal = malloc(npt*sizeof(PetscScalar));
yarray = malloc(pmax*sizeof(PetscScalar));
NzindJ = malloc(pmax*npt*sizeof(PetscInt));
idx = malloc(pmax*npt*sizeof(PetscInt));
idy = malloc(pmax*npt*sizeof(PetscInt));
idp = malloc(pmax*sizeof(PetscInt));
*(ShiftX+0) = 0;
*(ShiftY+0) = 0;
*(ShiftX+1) = -dx2;
*(ShiftY+1) = -dy2;
*(ShiftX+2) = dx2;
*(ShiftY+2) = -dy2;
*(ShiftX+3) = -dx2;
*(ShiftY+3) = dy2;
*(ShiftX+4) = dy2;
*(ShiftY+4) = dx2;
//*(ShiftX+5) = 0;
//*(ShiftY+5) = -dy2;
//*(ShiftX+6) = -dx2;
//*(ShiftY+6) = 0;
//*(ShiftX+7) = dx2;
//*(ShiftY+7) = 0;
//*(ShiftX+8) = 0;
//*(ShiftY+9) = dy2;
for(i=0;i<npt*pmax;i++){ *(idy+i)=i; }
ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*pmax,idy,&isy);
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
l = 0; p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l;}else{puse=pmax;}
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
CX = dx2+i*dx;
CY = dy2+j*dy;
for(k=0;k<npt;k++){
*(CenterX+k) = CX + *(ShiftX+k);
*(CenterY+k) = CY + *(ShiftY+k);
InverseStandardMap((CenterX+k),(CenterY+k));
*(NzindJ+p*npt +k) = floor(*(CenterX+k)*n)*n + floor(*(CenterY+k)*n);
}
*(idp+p) = Istart*n+ l;
if(p>=puse-1){
ierr = ISCreateGeneralWithArray(PETSC_COMM_WORLD,npt*puse,NzindJ,&isx);CHKERRQ(ierr);
for(m=0;m<npt*puse;m++){ *(idy+m)=m; }
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*puse,idy,&isy);CHKERRQ(ierr);
ierr = VecScatterCreate(*vinda,isx,y0,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterEnd(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterDestroy(ctx);
ierr = VecGetArray(y0,&pty);CHKERRQ(ierr);
for(m=0;m<puse;m++){
for(m2=0;m2<npt;m2++){
*(yarray+m) = *(yarray+m)+*(pty+m*npt+m2);
}
*(yarray+m) = *(yarray+m)/npt;
}
VecRestoreArray(y0,&pty);
VecSetValues(*vindb,puse,idp,yarray,INSERT_VALUES);
for(m=0;m<pmax;m++){*(yarray+m) = 0; }
p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l-1;}else{puse=pmax;}
}else{p++;}
l++;
}
}
VecAssemblyBegin(*vindb);
VecAssemblyEnd(*vindb);
vindc = vindb;
vindb = vinda;
vinda = vindc;
//ierr = VecCopy(x,x0);CHKERRQ(ierr);
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(x0,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
free(CenterX);
free(CenterY);
free(ShiftX);
free(ShiftY);
free(x0array);
free(idx);
free(idy);
free(idp);
free(yarray);
free(NzindJ);
free(Mixnorm);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(y0);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
PetscErrorCode BearQuery(PetscInt s, PetscScalar c, Mat invL1, Mat invU1, Mat invL2, Mat invU2, Mat H12, Mat H21, Vec order, Vec or){
PetscErrorCode err;
PetscInt n1, n2, n;
PetscInt oseed;
PetscScalar val, one = 1.0;
Vec r = NULL;
Vec r1 = NULL, q1 = NULL, t1_1 = NULL, t1_2 = NULL, t1_3 = NULL, t1_4 = NULL, t1_5 = NULL; // dimension: n1
Vec r2 = NULL, q2 = NULL, q_tilda = NULL, t2_1 = NULL, t2_2 = NULL, t2_3 = NULL; // dimension: n2
err = MatGetSize(H12, &n1, &n2); CHKERRQ(err);
n = n1 + n2;
// err = PetscPrintf(PETSC_COMM_WORLD, "n1: %d, n2: %d\n", n1, n2); CHKERRQ(err);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n, &r); CHKERRQ(err);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n1, &q1); CHKERRQ(err);
err = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, n2, &q2); CHKERRQ(err);
err = VecSet(q1, 0); CHKERRQ(err);
err = VecSet(q2, 0); CHKERRQ(err);
s = s - 1; // shift -1 for zero-based index
err = VecGetValues(order, 1, &s, &val); CHKERRQ(err);
oseed = (PetscInt) val;
// err = PetscPrintf(PETSC_COMM_WORLD, "Given seed: %d, Reorered seed: %d (0 ~ n-1)\n", s, oseed); CHKERRQ(err);
if(oseed < n1){
err = VecSetValues(q1, 1, &oseed, &one, INSERT_VALUES); CHKERRQ(err);
}else{
oseed = oseed - n1;
err = VecSetValues(q2, 1, &oseed, &one, INSERT_VALUES); CHKERRQ(err);
//err = printVecSum(q2);
}
err = VecAssemblyBegin(q1); CHKERRQ(err);
err = VecAssemblyBegin(q2); CHKERRQ(err);
err = VecAssemblyEnd(q1); CHKERRQ(err);
err = VecAssemblyEnd(q2); CHKERRQ(err);
err = VecDuplicate(q1, &r1); CHKERRQ(err);
err = VecDuplicate(q1, &t1_1); CHKERRQ(err);
err = VecDuplicate(q1, &t1_2); CHKERRQ(err);
err = VecDuplicate(q1, &t1_3); CHKERRQ(err);
err = VecDuplicate(q1, &t1_4); CHKERRQ(err);
err = VecDuplicate(q1, &t1_5); CHKERRQ(err);
err = VecDuplicate(q2, &r2); CHKERRQ(err);
err = VecDuplicate(q2, &q_tilda); CHKERRQ(err);
err = VecDuplicate(q2, &t2_1); CHKERRQ(err);
err = VecDuplicate(q2, &t2_2); CHKERRQ(err);
err = VecDuplicate(q2, &t2_3); CHKERRQ(err);
// Start matrix-vec multiplications
err = MatMult(invL1, q1, t1_1); CHKERRQ(err);
err = MatMult(invU1, t1_1, t1_2); CHKERRQ(err);
err = MatMult(H21, t1_2, t2_1); CHKERRQ(err);
err = VecAXPBYPCZ(q_tilda, 1.0, -1.0, 0.0, q2, t2_1); CHKERRQ(err);
err = MatMult(invL2, q_tilda, t2_2); CHKERRQ(err);
err = MatMult(invU2, t2_2, r2); CHKERRQ(err);
err = MatMult(H12, r2, t1_3); CHKERRQ(err);
err = VecAXPBYPCZ(t1_4, 1.0, -1.0, 0.0, q1, t1_3); CHKERRQ(err);
err = MatMult(invL1, t1_4, t1_5); CHKERRQ(err);
err = MatMult(invU1, t1_5, r1); CHKERRQ(err);
//err = printVecSum(r1);
//err = VecView(r2, PETSC_VIEWER_STDOUT_WORLD);
// Concatenate r1 and r2
err = VecMerge(r1, r2, r); CHKERRQ(err);
err = VecScale(r, c); CHKERRQ(err);
//err = VecView(r, PETSC_VIEWER_STDOUT_WORLD);
//err = VecDuplicate(r, &or); CHKERRQ(err);
err = VecReorder(r, order, or); CHKERRQ(err);
//err = VecView(or, PETSC_VIEWER_STDOUT_WORLD);
err = VecDestroy(&r); CHKERRQ(err);
err = VecDestroy(&r1); CHKERRQ(err);
err = VecDestroy(&q1); CHKERRQ(err);
err = VecDestroy(&t1_1); CHKERRQ(err);
err = VecDestroy(&t1_2); CHKERRQ(err);
err = VecDestroy(&t1_3); CHKERRQ(err);
err = VecDestroy(&t1_4); CHKERRQ(err);
err = VecDestroy(&t1_5); CHKERRQ(err);
err = VecDestroy(&r2); CHKERRQ(err);
err = VecDestroy(&q2); CHKERRQ(err);
err = VecDestroy(&q_tilda); CHKERRQ(err);
err = VecDestroy(&t2_1); CHKERRQ(err);
err = VecDestroy(&t2_2); CHKERRQ(err);
err = VecDestroy(&t2_3); CHKERRQ(err);
return err;
}
int main(int argc, char **argv)
{
/* -------Initialize and Get the parameters from command line ------*/
PetscInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
PetscPrintf(PETSC_COMM_WORLD,"--------Initializing------ \n");
PetscErrorCode ierr;
PetscBool flg;
int myrank;
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
if(myrank==0)
mma_verbose=1;
/*-------------------------------------------------*/
int Mx,My,Mz,Mzslab, Npmlx,Npmly,Npmlz,DegFree, anisotropic;
PetscOptionsGetInt(PETSC_NULL,"-Nx",&Nx,&flg); MyCheckAndOutputInt(flg,Nx,"Nx","Nx");
PetscOptionsGetInt(PETSC_NULL,"-Ny",&Ny,&flg); MyCheckAndOutputInt(flg,Ny,"Ny","Nx");
PetscOptionsGetInt(PETSC_NULL,"-Nz",&Nz,&flg); MyCheckAndOutputInt(flg,Nz,"Nz","Nz");
PetscOptionsGetInt(PETSC_NULL,"-Mx",&Mx,&flg); MyCheckAndOutputInt(flg,Mx,"Mx","Mx");
PetscOptionsGetInt(PETSC_NULL,"-My",&My,&flg); MyCheckAndOutputInt(flg,My,"My","My");
PetscOptionsGetInt(PETSC_NULL,"-Mz",&Mz,&flg); MyCheckAndOutputInt(flg,Mz,"Mz","Mz");
PetscOptionsGetInt(PETSC_NULL,"-Mzslab",&Mzslab,&flg); MyCheckAndOutputInt(flg,Mzslab,"Mzslab","Mzslab");
PetscOptionsGetInt(PETSC_NULL,"-Npmlx",&Npmlx,&flg); MyCheckAndOutputInt(flg,Npmlx,"Npmlx","Npmlx");
PetscOptionsGetInt(PETSC_NULL,"-Npmly",&Npmly,&flg); MyCheckAndOutputInt(flg,Npmly,"Npmly","Npmly");
PetscOptionsGetInt(PETSC_NULL,"-Npmlz",&Npmlz,&flg); MyCheckAndOutputInt(flg,Npmlz,"Npmlz","Npmlz");
Nxyz = Nx*Ny*Nz;
// if anisotropic !=0, Degree of Freedom = 3*Mx*My*Mz; else DegFree = Mx*My*Mz;
PetscOptionsGetInt(PETSC_NULL,"-anisotropic",&anisotropic,&flg);
if(!flg) anisotropic = 0; // by default, it is isotropc.
DegFree = (anisotropic ? 3 : 1 )*Mx*My*((Mzslab==0)?Mz:1);
PetscPrintf(PETSC_COMM_WORLD," the Degree of Freedoms is %d \n ", DegFree);
int DegFreeAll=DegFree+1;
PetscPrintf(PETSC_COMM_WORLD," the Degree of Freedoms ALL is %d \n ", DegFreeAll);
int BCPeriod, Jdirection, Jdirectiontwo, LowerPML;
int bx[2], by[2], bz[2];
PetscOptionsGetInt(PETSC_NULL,"-BCPeriod",&BCPeriod,&flg); MyCheckAndOutputInt(flg,BCPeriod,"BCPeriod","BCPeriod given");
PetscOptionsGetInt(PETSC_NULL,"-Jdirection",&Jdirection,&flg); MyCheckAndOutputInt(flg,Jdirection,"Jdirection","Diapole current direction");
PetscOptionsGetInt(PETSC_NULL,"-Jdirectiontwo",&Jdirectiontwo,&flg); MyCheckAndOutputInt(flg,Jdirectiontwo,"Jdirectiontwo","Diapole current direction for source two");
PetscOptionsGetInt(PETSC_NULL,"-LowerPML",&LowerPML,&flg); MyCheckAndOutputInt(flg,LowerPML,"LowerPML","PML in the lower xyz boundary");
PetscOptionsGetInt(PETSC_NULL,"-bxl",bx,&flg); MyCheckAndOutputInt(flg,bx[0],"bxl","BC at x lower");
PetscOptionsGetInt(PETSC_NULL,"-bxu",bx+1,&flg); MyCheckAndOutputInt(flg,bx[1],"bxu","BC at x upper");
PetscOptionsGetInt(PETSC_NULL,"-byl",by,&flg); MyCheckAndOutputInt(flg,by[0],"byl","BC at y lower");
PetscOptionsGetInt(PETSC_NULL,"-byu",by+1,&flg); MyCheckAndOutputInt(flg,by[1],"byu","BC at y upper");
PetscOptionsGetInt(PETSC_NULL,"-bzl",bz,&flg); MyCheckAndOutputInt(flg,bz[0],"bzl","BC at z lower");
PetscOptionsGetInt(PETSC_NULL,"-bzu",bz+1,&flg); MyCheckAndOutputInt(flg,bz[1],"bzu","BC at z upper");
double epssub, RRT, sigmax, sigmay, sigmaz ;
PetscOptionsGetReal(PETSC_NULL,"-hx",&hx,&flg); MyCheckAndOutputDouble(flg,hx,"hx","hx");
hy = hx;
hz = hx;
hxyz = (Nz==1)*hx*hy + (Nz>1)*hx*hy*hz;
double omega, omegaone, omegatwo, wratio;
PetscOptionsGetReal(PETSC_NULL,"-omega",&omega,&flg); MyCheckAndOutputDouble(flg,omega,"omega","omega");
PetscOptionsGetReal(PETSC_NULL,"-wratio",&wratio,&flg); MyCheckAndOutputDouble(flg,wratio,"wratio","wratio");
omegaone=omega;
omegatwo=wratio*omega;
PetscPrintf(PETSC_COMM_WORLD,"---omegaone is %.16e and omegatwo is %.16e ---\n",omegaone, omegatwo);
PetscOptionsGetReal(PETSC_NULL,"-Qabs",&Qabs,&flg);
if (flg && Qabs>1e+15)
Qabs=1.0/0.0;
MyCheckAndOutputDouble(flg,Qabs,"Qabs","Qabs");
PetscOptionsGetReal(PETSC_NULL,"-epsair",&epsair,&flg); MyCheckAndOutputDouble(flg,epsair,"epsair","epsair");
PetscOptionsGetReal(PETSC_NULL,"-epssub",&epssub,&flg); MyCheckAndOutputDouble(flg,epssub,"epssub","epssub");
PetscOptionsGetReal(PETSC_NULL,"-RRT",&RRT,&flg); MyCheckAndOutputDouble(flg,RRT,"RRT","RRT given");
sigmax = pmlsigma(RRT,Npmlx*hx);
sigmay = pmlsigma(RRT,Npmly*hy);
sigmaz = pmlsigma(RRT,Npmlz*hz);
PetscPrintf(PETSC_COMM_WORLD,"----sigmax is %.12e \n",sigmax);
PetscPrintf(PETSC_COMM_WORLD,"----sigmay is %.12e \n",sigmay);
PetscPrintf(PETSC_COMM_WORLD,"----sigmaz is %.12e \n",sigmaz);
char initialdata[PETSC_MAX_PATH_LEN]; //filenameComm[PETSC_MAX_PATH_LEN];
PetscOptionsGetString(PETSC_NULL,"-initialdata",initialdata,PETSC_MAX_PATH_LEN,&flg); MyCheckAndOutputChar(flg,initialdata,"initialdata","Inputdata file");
PetscOptionsGetString(PETSC_NULL,"-filenameComm",filenameComm,PETSC_MAX_PATH_LEN,&flg); MyCheckAndOutputChar(flg,filenameComm,"filenameComm","Output filenameComm");
// add cx, cy, cz to indicate where the diapole current is;
int cx, cy, cz;
PetscOptionsGetInt(PETSC_NULL,"-cx",&cx,&flg);
if (!flg)
{cx=(LowerPML)*floor(Nx/2); PetscPrintf(PETSC_COMM_WORLD,"cx is %d by default \n",cx);}
else
{PetscPrintf(PETSC_COMM_WORLD,"the current poisiont cx is %d \n",cx);}
PetscOptionsGetInt(PETSC_NULL,"-cy",&cy,&flg);
if (!flg)
{cy=(LowerPML)*floor(Ny/2); PetscPrintf(PETSC_COMM_WORLD,"cy is %d by default \n",cy);}
else
{PetscPrintf(PETSC_COMM_WORLD,"the current poisiont cy is %d \n",cy);}
PetscOptionsGetInt(PETSC_NULL,"-cz",&cz,&flg);
if (!flg)
{cz=(LowerPML)*floor(Nz/2); PetscPrintf(PETSC_COMM_WORLD,"cz is %d by default \n",cz);}
else
{PetscPrintf(PETSC_COMM_WORLD,"the current poisiont cz is %d \n",cz);}
posj = (cx*Ny+ cy)*Nz + cz;
PetscPrintf(PETSC_COMM_WORLD,"the posj is %d \n. ", posj);
int fixpteps;
PetscOptionsGetInt(PETSC_NULL,"-fixpteps",&fixpteps,&flg); MyCheckAndOutputInt(flg,fixpteps,"fixpteps","fixpteps");
// Get minapproach;
PetscOptionsGetInt(PETSC_NULL,"-minapproach",&minapproach,&flg); MyCheckAndOutputInt(flg,minapproach,"minapproach","minapproach");
// Get withepsinldos;
PetscOptionsGetInt(PETSC_NULL,"-withepsinldos",&withepsinldos,&flg); MyCheckAndOutputInt(flg,withepsinldos,"withepsinldos","withepsinldos");
// Get outputbase;
PetscOptionsGetInt(PETSC_NULL,"-outputbase",&outputbase,&flg); MyCheckAndOutputInt(flg,outputbase,"outputbase","outputbase");
// Get cavityverbose;
PetscOptionsGetInt(PETSC_NULL,"-cavityverbose",&cavityverbose,&flg);
if(!flg) cavityverbose=0;
PetscPrintf(PETSC_COMM_WORLD,"the cavity verbose is set as %d \n", cavityverbose);
// Get refinedldos;
PetscOptionsGetInt(PETSC_NULL,"-refinedldos",&refinedldos,&flg);
if(!flg) refinedldos=0;
PetscPrintf(PETSC_COMM_WORLD,"the refinedldos is set as %d \n", refinedldos);
// Get cmpwrhs;
int cmpwrhs;
PetscOptionsGetInt(PETSC_NULL,"-cmpwrhs",&cmpwrhs,&flg);
if(!flg) cmpwrhs=0;
PetscPrintf(PETSC_COMM_WORLD,"the cmpwrhs is set as %d \n", cmpwrhs);
// Get lrzsqr;
PetscOptionsGetInt(PETSC_NULL,"-lrzsqr",&lrzsqr,&flg);
if(!flg) lrzsqr=0;
PetscPrintf(PETSC_COMM_WORLD,"the lrzsqr is set as %d \n", lrzsqr);
// Get newQdef;
PetscOptionsGetInt(PETSC_NULL,"-newQdef",&newQdef,&flg);
if(!flg) newQdef=0;
PetscPrintf(PETSC_COMM_WORLD,"the newQdef is set as %d \n", newQdef);
/*--------------------------------------------------------*/
/*--------------------------------------------------------*/
/*---------- Set the current source---------*/
//Mat D; //ImaginaryIMatrix;
ImagIMat(PETSC_COMM_WORLD, &D,6*Nxyz);
Vec J;
ierr = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, 6*Nxyz, &J);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) J, "Source");CHKERRQ(ierr);
VecSet(J,0.0); //initialization;
if (Jdirection == 1)
SourceSingleSetX(PETSC_COMM_WORLD, J, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirection ==2)
SourceSingleSetY(PETSC_COMM_WORLD, J, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirection == 3)
SourceSingleSetZ(PETSC_COMM_WORLD, J, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else
PetscPrintf(PETSC_COMM_WORLD," Please specify correct direction of current: x (1) , y (2) or z (3)\n ");
Vec Jtwo;
ierr = VecDuplicate(J, &Jtwo);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) Jtwo, "Sourcetwo");CHKERRQ(ierr);
VecSet(Jtwo,0.0); //initialization;
if (Jdirectiontwo == 1)
SourceSingleSetX(PETSC_COMM_WORLD, Jtwo, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirectiontwo ==2)
SourceSingleSetY(PETSC_COMM_WORLD, Jtwo, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirectiontwo == 3)
SourceSingleSetZ(PETSC_COMM_WORLD, Jtwo, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else
PetscPrintf(PETSC_COMM_WORLD," Please specify correct direction of current two: x (1) , y (2) or z (3)\n ");
//Vec b; // b= i*omega*J;
Vec bone, btwo;
ierr = VecDuplicate(J,&b);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) b, "rhsone");CHKERRQ(ierr);
ierr = VecDuplicate(J,&bone);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) bone, "rhsone");CHKERRQ(ierr);
ierr = VecDuplicate(Jtwo,&btwo);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) btwo, "rhstwo");CHKERRQ(ierr);
if (cmpwrhs==0)
{
ierr = MatMult(D,J,b);CHKERRQ(ierr);
ierr = MatMult(D,Jtwo,btwo);CHKERRQ(ierr);
VecCopy(b,bone);
VecScale(bone,omegaone);
VecScale(btwo,omegatwo);
VecScale(b,omega);
}
else
{
double complex cmpiomega;
cmpiomega = cpow(1+I/Qabs,newQdef+1);
double sqrtiomegaR = -omega*cimag(csqrt(cmpiomega));
double sqrtiomegaI = omega*creal(csqrt(cmpiomega));
PetscPrintf(PETSC_COMM_WORLD,"the real part of sqrt cmpomega is %g and imag sqrt is % g ", sqrtiomegaR, sqrtiomegaI);
Vec tmpi;
ierr = VecDuplicate(J,&tmpi);
VecSet(b,0.0);
VecSet(tmpi,0.0);
CmpVecScale(J,b,sqrtiomegaR,sqrtiomegaI,D,tmpi);
VecDestroy(&tmpi);
}
/*-------Get the weight vector ------------------*/
//Vec weight;
ierr = VecDuplicate(J,&weight); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) weight, "weight");CHKERRQ(ierr);
if(LowerPML==0)
GetWeightVec(weight, Nx, Ny,Nz); // new code handles both 3D and 2D;
else
VecSet(weight,1.0);
Vec weightedJ;
ierr = VecDuplicate(J,&weightedJ); CHKERRQ(ierr);
ierr = VecPointwiseMult(weightedJ,J,weight);
ierr = PetscObjectSetName((PetscObject) weightedJ, "weightedJ");CHKERRQ(ierr);
Vec weightedJtwo;
ierr = VecDuplicate(Jtwo,&weightedJtwo); CHKERRQ(ierr);
ierr = VecPointwiseMult(weightedJtwo,Jtwo,weight);
ierr = PetscObjectSetName((PetscObject) weightedJtwo, "weightedJtwo");CHKERRQ(ierr);
//Vec vR;
ierr = VecDuplicate(J,&vR); CHKERRQ(ierr);
GetRealPartVec(vR, 6*Nxyz);
// VecFReal;
if (lrzsqr)
{ ierr = VecDuplicate(J,&epsFReal); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsFReal, "epsFReal");CHKERRQ(ierr);
if (newQdef==0)
{
sqrtomegaI = omega*cimag(csqrt(1+I/Qabs));
PetscPrintf(PETSC_COMM_WORLD,"the real part of sqrt cmpomega is %g and imag sqrt is % g ", omega*creal(csqrt(1+I/Qabs)), sqrtomegaI);
betar = 2*sqrtomegaI;
betai = betar/Qabs;
}
else
{
double gamma;
gamma = omega/Qabs;
betar = 2*gamma*(1-1.0/pow(Qabs,2));
betai = 2*gamma*(2.0/Qabs);
}
ierr = VecDuplicate(J,&nb); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) nb, "nb"); CHKERRQ(ierr);
ierr = VecDuplicate(J,&y); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "y"); CHKERRQ(ierr);
ierr = VecDuplicate(J,&xsqr); CHKERRQ(ierr); // xsqr = x*x;
ierr = PetscObjectSetName((PetscObject) xsqr, "xsqr"); CHKERRQ(ierr);
CongMat(PETSC_COMM_WORLD, &C, 6*Nxyz);
}
/*----------- Define PML muinv vectors */
Vec muinvpml;
MuinvPMLFull(PETSC_COMM_SELF, &muinvpml,Nx,Ny,Nz,Npmlx,Npmly,Npmlz,sigmax,sigmay,sigmaz,omega, LowerPML);
//double *muinv;
muinv = (double *) malloc(sizeof(double)*6*Nxyz);
int add=0;
AddMuAbsorption(muinv,muinvpml,Qabs,add);
ierr = VecDestroy(&muinvpml); CHKERRQ(ierr);
/*---------- Define PML eps vectors: epspml---------- */
Vec epspml; //epspmlQ, epscoef;
ierr = VecDuplicate(J,&epspml);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epspml,"EpsPMLFull"); CHKERRQ(ierr);
EpsPMLFull(PETSC_COMM_WORLD, epspml,Nx,Ny,Nz,Npmlx,Npmly,Npmlz,sigmax,sigmay,sigmaz,omega, LowerPML);
ierr = VecDuplicate(J,&epspmlQ);CHKERRQ(ierr);
Vec epscoefone, epscoeftwo;
ierr = VecDuplicate(J,&epscoefone);CHKERRQ(ierr);
ierr = VecDuplicate(J,&epscoeftwo);CHKERRQ(ierr);
// compute epspmlQ,epscoef;
EpsCombine(D, weight, epspml, epspmlQ, epscoefone, Qabs, omegaone);
EpsCombine(D, weight, epspml, epspmlQ, epscoeftwo, Qabs, omegatwo);
/*--------- Setup the interp matrix ----------------------- */
/* for a samll eps block, interp it into yee-lattice. The interp matrix A and PML epspml only need to generated once;*/
//Mat A;
//new routine for myinterp;
myinterp(PETSC_COMM_WORLD, &A, Nx,Ny,Nz, LowerPML*floor((Nx-Mx)/2),LowerPML*floor((Ny-My)/2),LowerPML*floor((Nz-Mz)/2), Mx,My,Mz,Mzslab, anisotropic); // LoweerPML*Npmlx,..,.., specify where the interp starts;
//Vec epsSReal, epsgrad, vgrad; // create compatiable vectors with A.
ierr = MatGetVecs(A,&epsSReal, &epsgrad); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsgrad, "epsgrad");CHKERRQ(ierr);
ierr = VecDuplicate(epsSReal, &vgrad); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsSReal, "epsSReal");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) vgrad, "vgrad");CHKERRQ(ierr);
/*---------Setup the epsmedium vector----------------*/
//Vec epsmedium;
ierr = VecDuplicate(J,&epsmedium); CHKERRQ(ierr);
GetMediumVec(epsmedium,Nz,Mz,epsair,epssub);
/*--------- Setup the finitie difference matrix-------------*/
//Mat M;
MoperatorGeneral(PETSC_COMM_WORLD, &M, Nx,Ny,Nz,hx,hy,hz, bx, by, bz,muinv,BCPeriod);
free(muinv);
/*--------Setup the KSP variables ---------------*/
KSP kspone;
PC pcone;
ierr = KSPCreate(PETSC_COMM_WORLD,&kspone);CHKERRQ(ierr);
//ierr = KSPSetType(ksp, KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetType(kspone, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGetPC(kspone,&pcone);CHKERRQ(ierr);
ierr = PCSetType(pcone,PCLU);CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(pcone,MATSOLVERPASTIX);CHKERRQ(ierr);
ierr = PCSetFromOptions(pcone);
int maxkspit = 20;
ierr = KSPSetTolerances(kspone,1e-14,PETSC_DEFAULT,PETSC_DEFAULT,maxkspit);CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspone);CHKERRQ(ierr);
KSP ksptwo;
PC pctwo;
ierr = KSPCreate(PETSC_COMM_WORLD,&ksptwo);CHKERRQ(ierr);
//ierr = KSPSetType(ksp, KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetType(ksptwo, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGetPC(ksptwo,&pctwo);CHKERRQ(ierr);
ierr = PCSetType(pctwo,PCLU);CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(pctwo,MATSOLVERPASTIX);CHKERRQ(ierr);
ierr = PCSetFromOptions(pctwo);
ierr = KSPSetTolerances(ksptwo,1e-14,PETSC_DEFAULT,PETSC_DEFAULT,maxkspit);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksptwo);CHKERRQ(ierr);
/*--------- Create the space for solution vector -------------*/
//Vec x;
ierr = VecDuplicate(J,&x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Solution");CHKERRQ(ierr);
/*----------- Create the space for final eps -------------*/
//Vec epsC, epsCi, epsP;
ierr = VecDuplicate(J,&epsC);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsC, "EpsC");CHKERRQ(ierr);
ierr = VecDuplicate(J,&epsCi);CHKERRQ(ierr);
ierr = VecDuplicate(J,&epsP);CHKERRQ(ierr);
ierr = VecSet(epsP,0.0); CHKERRQ(ierr);
ierr = VecAssemblyBegin(epsP); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epsP); CHKERRQ(ierr);
/*------------ Create space used in the solver ------------*/
//Vec vgradlocal,tmp, tmpa,tmpb;
ierr = VecCreateSeq(PETSC_COMM_SELF, DegFree, &vgradlocal); CHKERRQ(ierr);
ierr = VecDuplicate(J,&tmp); CHKERRQ(ierr);
ierr = VecDuplicate(J,&tmpa); CHKERRQ(ierr);
ierr = VecDuplicate(J,&tmpb); CHKERRQ(ierr);
// Vec pickposvec; this vector is zero except that first entry is one;
if (withepsinldos)
{ ierr = VecDuplicate(J,&pickposvec); CHKERRQ(ierr);
ierr = VecSet(pickposvec,0.0); CHKERRQ(ierr);
ierr = VecSetValue(pickposvec,posj+Jdirection*Nxyz,1.0,INSERT_VALUES);
VecAssemblyBegin(pickposvec);
VecAssemblyEnd(pickposvec);
}
/*------------ Create scatter used in the solver -----------*/
//VecScatter scatter;
//IS from, to;
ierr =ISCreateStride(PETSC_COMM_SELF,DegFree,0,1,&from); CHKERRQ(ierr);
ierr =ISCreateStride(PETSC_COMM_SELF,DegFree,0,1,&to); CHKERRQ(ierr);
/*-------------Read the input file -------------------------*/
double *epsoptAll;
epsoptAll = (double *) malloc(DegFreeAll*sizeof(double));
FILE *ptf;
ptf = fopen(initialdata,"r");
PetscPrintf(PETSC_COMM_WORLD,"reading from input files \n");
int i;
// set the dielectric at the center is fixed, and alwyas high
//epsopt[0]=myub; is defined below near lb and ub;
for (i=0;i<DegFree;i++)
{ //PetscPrintf(PETSC_COMM_WORLD,"current eps reading is %lf \n",epsopt[i]);
fscanf(ptf,"%lf",&epsoptAll[i]);
}
epsoptAll[DegFreeAll-1]=0; //initialize auxiliary variable;
fclose(ptf);
/*----declare these data types, althought they may not be used for job 2 -----------------*/
double mylb,myub, *lb=NULL, *ub=NULL;
int maxeval, maxtime, mynloptalg;
double maxf;
nlopt_opt opt;
nlopt_result result;
/*--------------------------------------------------------------*/
/*----Now based on Command Line, Do the corresponding job----*/
/*----------------------------------------------------------------*/
//int Job; set Job to be gloabl variables;
PetscOptionsGetInt(PETSC_NULL,"-Job",&Job,&flg); MyCheckAndOutputInt(flg,Job,"Job","The Job indicator you set");
int numofvar=(Job==1)*DegFreeAll + (Job==3);
/*-------- convert the epsopt array to epsSReal (if job!=optmization) --------*/
if (Job==2 || Job ==3)
{
// copy epsilon from file to epsSReal; (different from FindOpt.c, because epsilon is not degree-of-freedoms in computeQ.
// i) create a array to read file (done above in epsopt); ii) convert the array to epsSReal;
int ns, ne;
ierr = VecGetOwnershipRange(epsSReal,&ns,&ne);
for(i=ns;i<ne;i++)
{ ierr=VecSetValue(epsSReal,i,epsoptAll[i],INSERT_VALUES);
CHKERRQ(ierr); }
if(withepsinldos)
{ epsatinterest = epsoptAll[cx*Ny*Nz + cy*Nz + cz] + epsair;
PetscPrintf(PETSC_COMM_WORLD, " the relative permitivity at the point of current is %.16e \n ",epsatinterest);}
ierr = VecAssemblyBegin(epsSReal); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epsSReal); CHKERRQ(ierr);
}
if (Job==1 || Job==3) // optimization bounds setup;
{
PetscOptionsGetInt(PETSC_NULL,"-maxeval",&maxeval,&flg); MyCheckAndOutputInt(flg,maxeval,"maxeval","max number of evaluation");
PetscOptionsGetInt(PETSC_NULL,"-maxtime",&maxtime,&flg); MyCheckAndOutputInt(flg,maxtime,"maxtime","max time of evaluation");
PetscOptionsGetInt(PETSC_NULL,"-mynloptalg",&mynloptalg,&flg); MyCheckAndOutputInt(flg,mynloptalg,"mynloptalg","The algorithm used ");
PetscOptionsGetReal(PETSC_NULL,"-mylb",&mylb,&flg); MyCheckAndOutputDouble(flg,mylb,"mylb","optimization lb");
PetscOptionsGetReal(PETSC_NULL,"-myub",&myub,&flg); MyCheckAndOutputDouble(flg,myub,"myub","optimization ub");
lb = (double *) malloc(numofvar*sizeof(double));
ub = (double *) malloc(numofvar*sizeof(double));
// the dielectric constant at center is fixed!
for(i=0;i<numofvar;i++)
{
lb[i] = mylb;
ub[i] = myub;
} //initial guess, lower bounds, upper bounds;
// set lower and upper bounds for auxiliary variable;
lb[numofvar-1]=0;
ub[numofvar-1]=1.0/0.0;
//fix the dielectric at the center to be high for topology optimization;
if (Job==1 && fixpteps==1)
{
epsoptAll[0]=myub;
lb[0]=myub;
ub[0]=myub;
}
opt = nlopt_create(mynloptalg, numofvar);
myfundatatypeshg data[2] = {{omegaone, bone, weightedJ, epscoefone,kspone},{omegatwo, btwo, weightedJtwo, epscoeftwo,ksptwo}};
nlopt_add_inequality_constraint(opt,ldosconstraint, &data[0], 1e-8);
nlopt_add_inequality_constraint(opt,ldosconstraint, &data[1], 1e-8);
nlopt_set_lower_bounds(opt,lb);
nlopt_set_upper_bounds(opt,ub);
nlopt_set_maxeval(opt,maxeval);
nlopt_set_maxtime(opt,maxtime);
/*add functionality to choose local optimizer; */
int mynloptlocalalg;
nlopt_opt local_opt;
PetscOptionsGetInt(PETSC_NULL,"-mynloptlocalalg",&mynloptlocalalg,&flg); MyCheckAndOutputInt(flg,mynloptlocalalg,"mynloptlocalalg","The local optimization algorithm used ");
if (mynloptlocalalg)
{
local_opt=nlopt_create(mynloptlocalalg,numofvar);
nlopt_set_ftol_rel(local_opt, 1e-14);
nlopt_set_maxeval(local_opt,100000);
nlopt_set_local_optimizer(opt,local_opt);
}
}
switch (Job)
{
case 1:
{
if (minapproach)
nlopt_set_min_objective(opt,maxminobjfun,NULL);// NULL: no data to be passed because of global variables;
else
nlopt_set_max_objective(opt,maxminobjfun,NULL);
result = nlopt_optimize(opt,epsoptAll,&maxf);
}
break;
case 2 : //AnalyzeStructure
{
int Linear, Eig, maxeigit;
PetscOptionsGetInt(PETSC_NULL,"-Linear",&Linear,&flg); MyCheckAndOutputInt(flg,Linear,"Linear","Linear solver indicator");
PetscOptionsGetInt(PETSC_NULL,"-Eig",&Eig,&flg); MyCheckAndOutputInt(flg,Eig,"Eig","Eig solver indicator");
PetscOptionsGetInt(PETSC_NULL,"-maxeigit",&maxeigit,&flg); MyCheckAndOutputInt(flg,maxeigit,"maxeigit","maximum number of Eig solver iterations is");
/*----------------------------------*/
//EigenSolver(Linear, Eig, maxeigit);
/*----------------------------------*/
OutputVec(PETSC_COMM_WORLD, weight,filenameComm, "weight.m");
}
break;
default:
PetscPrintf(PETSC_COMM_WORLD,"--------Interesting! You're doing nothing!--------\n ");
}
if(Job==1 || Job==3)
{
/* print the optimization parameters */
#if 0
double xrel, frel, fabs;
// double *xabs;
frel=nlopt_get_ftol_rel(opt);
fabs=nlopt_get_ftol_abs(opt);
xrel=nlopt_get_xtol_rel(opt);
PetscPrintf(PETSC_COMM_WORLD,"nlopt frel is %g \n",frel);
PetscPrintf(PETSC_COMM_WORLD,"nlopt fabs is %g \n",fabs);
PetscPrintf(PETSC_COMM_WORLD,"nlopt xrel is %g \n",xrel);
//nlopt_result nlopt_get_xtol_abs(const nlopt_opt opt, double *tol);
#endif
/*--------------*/
if (result < 0) {
PetscPrintf(PETSC_COMM_WORLD,"nlopt failed! \n", result);
}
else {
PetscPrintf(PETSC_COMM_WORLD,"found extremum %0.16e\n", minapproach?1.0/maxf:maxf);
}
PetscPrintf(PETSC_COMM_WORLD,"nlopt returned value is %d \n", result);
if(Job==1)
{ //OutputVec(PETSC_COMM_WORLD, epsopt,filenameComm, "epsopt.m");
//OutputVec(PETSC_COMM_WORLD, epsgrad,filenameComm, "epsgrad.m");
//OutputVec(PETSC_COMM_WORLD, vgrad,filenameComm, "vgrad.m");
//OutputVec(PETSC_COMM_WORLD, x,filenameComm, "x.m");
int rankA;
MPI_Comm_rank(PETSC_COMM_WORLD, &rankA);
if(rankA==0)
{
ptf = fopen(strcat(filenameComm,"epsopt.txt"),"w");
for (i=0;i<DegFree;i++)
fprintf(ptf,"%0.16e \n",epsoptAll[i]);
fclose(ptf);
PetscPrintf(PETSC_COMM_WORLD,"the t parameter is %.8e \n",epsoptAll[DegFreeAll-1]);
}
}
nlopt_destroy(opt);
}
ierr = PetscPrintf(PETSC_COMM_WORLD,"--------Done!--------\n ");CHKERRQ(ierr);
/*------------------------------------*/
/* ----------------------Destroy Vecs and Mats----------------------------*/
free(epsoptAll);
free(lb);
free(ub);
ierr = VecDestroy(&J); CHKERRQ(ierr);
ierr = VecDestroy(&b); CHKERRQ(ierr);
ierr = VecDestroy(&weight); CHKERRQ(ierr);
ierr = VecDestroy(&weightedJ); CHKERRQ(ierr);
ierr = VecDestroy(&vR); CHKERRQ(ierr);
ierr = VecDestroy(&epspml); CHKERRQ(ierr);
ierr = VecDestroy(&epspmlQ); CHKERRQ(ierr);
ierr = VecDestroy(&epsSReal); CHKERRQ(ierr);
ierr = VecDestroy(&epsgrad); CHKERRQ(ierr);
ierr = VecDestroy(&vgrad); CHKERRQ(ierr);
ierr = VecDestroy(&epsmedium); CHKERRQ(ierr);
ierr = VecDestroy(&epsC); CHKERRQ(ierr);
ierr = VecDestroy(&epsCi); CHKERRQ(ierr);
ierr = VecDestroy(&epsP); CHKERRQ(ierr);
ierr = VecDestroy(&x); CHKERRQ(ierr);
ierr = VecDestroy(&vgradlocal);CHKERRQ(ierr);
ierr = VecDestroy(&tmp); CHKERRQ(ierr);
ierr = VecDestroy(&tmpa); CHKERRQ(ierr);
ierr = VecDestroy(&tmpb); CHKERRQ(ierr);
ierr = MatDestroy(&A); CHKERRQ(ierr);
ierr = MatDestroy(&D); CHKERRQ(ierr);
ierr = MatDestroy(&M); CHKERRQ(ierr);
ierr = VecDestroy(&epscoefone); CHKERRQ(ierr);
ierr = VecDestroy(&epscoeftwo); CHKERRQ(ierr);
ierr = KSPDestroy(&kspone);CHKERRQ(ierr);
ierr = KSPDestroy(&ksptwo);CHKERRQ(ierr);
ISDestroy(&from);
ISDestroy(&to);
if (withepsinldos)
{ierr=VecDestroy(&pickposvec); CHKERRQ(ierr);}
if (lrzsqr)
{
ierr=VecDestroy(&epsFReal); CHKERRQ(ierr);
ierr=VecDestroy(&xsqr); CHKERRQ(ierr);
ierr=VecDestroy(&y); CHKERRQ(ierr);
ierr=VecDestroy(&nb); CHKERRQ(ierr);
ierr=MatDestroy(&C); CHKERRQ(ierr);
}
ierr = VecDestroy(&bone); CHKERRQ(ierr);
ierr = VecDestroy(&btwo); CHKERRQ(ierr);
ierr = VecDestroy(&Jtwo); CHKERRQ(ierr);
/*------------ finalize the program -------------*/
{
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
//if (rank == 0) fgetc(stdin);
MPI_Barrier(PETSC_COMM_WORLD);
}
ierr = PetscFinalize(); CHKERRQ(ierr);
return 0;
}
