void linearSystemPETSc<scalar>::zeroRightHandSide()
{
if (_isAllocated) {
_try(VecAssemblyBegin(_b));
_try(VecAssemblyEnd(_b));
_try(VecZeroEntries(_b));
}
}
void linearSystemPETSc<scalar>::zeroSolution()
{
if (_isAllocated) {
_try(VecAssemblyBegin(_x));
_try(VecAssemblyEnd(_x));
_try(VecZeroEntries(_x));
}
}
void linearSystemPETSc<scalar>::clear()
{
if(_isAllocated){
_try(MatDestroy(&_a));
_try(VecDestroy(&_x));
_try(VecDestroy(&_b));
}
_isAllocated = false;
}
void linearSystemPETSc<scalar>::getFromSolution(int row, scalar &val) const
{
#if defined(PETSC_USE_COMPLEX)
PetscScalar *tmp;
_try(VecGetArray(_x, &tmp));
PetscScalar s = tmp[row];
_try(VecRestoreArray(_x, &tmp));
val = s.real();
#else
_try(VecGetValues(_x, 1, &row, &val));
#endif
}
void linearSystemPETSc<scalar>::getFromRightHandSide(int row, scalar &val) const
{
#if defined(PETSC_USE_COMPLEX)
PetscScalar *tmp;
_try(VecGetArray(_b, &tmp));
PetscScalar s = tmp[row];
_try(VecRestoreArray(_b, &tmp));
// FIXME specialize this routine
val = s.real();
#else
_try(VecGetValues(_b, 1, &row, &val));
#endif
}
void linearSystemPETSc<scalar>::printMatlab(const char *filename) const
{
_try(MatAssemblyBegin(_a, MAT_FINAL_ASSEMBLY));
_try(MatAssemblyEnd(_a, MAT_FINAL_ASSEMBLY));
_try(VecAssemblyBegin(_b));
_try(VecAssemblyEnd(_b));
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, filename, &viewer);
PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
MatView(_a, viewer);
PetscViewerDestroy(&viewer);
return;
}
int
gs_update_meta1_master (gs_grid_storage_t *gs, const container_id_t cID,
const char *m1)
{
int attempts = NB_ATTEMPTS_UPDATE_M1;
int _try (void)
{
GError *e = NULL;
if ((attempts--) <= 0) {
//leak memory, e not free
return 0;
}
/* tries a metacd resolution, and if it succeeds, clears the direct resolver */
if (resolver_metacd_is_up (gs->metacd_resolver)) {
if(resolver_metacd_set_meta1_master (gs->metacd_resolver, cID, m1, &e)) {
resolver_direct_clear (gs->direct_resolver);
return 1;
}
DEBUG("METACD error: META1 update failure. cause:\r\n\t%s",(e ? e->message : "?"));
if(NULL != e)
g_clear_error(&e);
}
if(!resolver_direct_set_meta1_master (gs->direct_resolver, cID, m1, &e))
DEBUG("META1 update failure. cause:\r\n\t%s",(e ? e->message : "?"));
if(NULL != e)
g_clear_error(&e);
return _try();
return 1;
}
double linearSystemPETSc<scalar>::normInfRightHandSide() const
{
PetscReal nor;
VecAssemblyBegin(_b);
VecAssemblyEnd(_b);
_try(VecNorm(_b, NORM_INFINITY, &nor));
return nor;
}
void linearSystemPETSc<scalar>::preAllocateEntries()
{
if (_entriesPreAllocated) return;
if (!_isAllocated) Msg::Fatal("system must be allocated first");
int blockSize = _getBlockSizeFromParameters();
std::vector<int> nByRowDiag (_localSize), nByRowOffDiag (_localSize);
if (_sparsity.getNbRows() == 0) {
PetscInt prealloc = 500;
PetscBool set;
PetscOptionsGetInt(PETSC_NULL, "-petsc_prealloc", &prealloc, &set);
prealloc = std::min(prealloc, _localSize);
nByRowDiag.resize(0);
nByRowDiag.resize(_localSize, prealloc);
} else {
for (int i = 0; i < _localSize; i++) {
int n;
const int *r = _sparsity.getRow(i, n);
for (int j = 0; j < n; j++) {
if (r[j] >= _localRowStart && r[j] < _localRowEnd)
nByRowDiag[i] ++;
else
nByRowOffDiag[i] ++;
}
}
_sparsity.clear();
}
//MatXAIJSetPreallocation is not available in petsc < 3.3
int commSize = 1;
MPI_Comm_size(_comm, &commSize);
if (commSize == 1){
if (blockSize == 1)
_try(MatSeqAIJSetPreallocation(_a, 0, &nByRowDiag[0]));
else
_try(MatSeqBAIJSetPreallocation(_a, blockSize, 0, &nByRowDiag[0]));
}
else {
if (blockSize == 1)
_try(MatMPIAIJSetPreallocation(_a, 0, &nByRowDiag[0], 0, &nByRowOffDiag[0]));
else
_try(MatMPIBAIJSetPreallocation(_a, blockSize, 0, &nByRowDiag[0], 0, &nByRowOffDiag[0]));
}
if (blockSize > 1)
_try(MatSetOption(_a, MAT_ROW_ORIENTED, PETSC_FALSE));
_entriesPreAllocated = true;
}
void linearSystemPETSc<scalar>::addToMatrix(int row, int col, const scalar &val)
{
if (!_entriesPreAllocated)
preAllocateEntries();
PetscInt i = row, j = col;
PetscScalar s = val;
_try(MatSetValues(_a, 1, &i, 1, &j, &s, ADD_VALUES));
_valuesNotAssembled = true;
}
void linearSystemPETSc<fullMatrix<double> >::addToSolution(int row, const fullMatrix<double> &val)
{
int blockSize;
_try(MatGetBlockSize(_a, &blockSize));
for (int ii = 0; ii < blockSize; ii++) {
PetscInt i = row * blockSize + ii;
PetscScalar v = val(ii, 0);
VecSetValues(_x, 1, &i, &v, ADD_VALUES);
}
}
void linearSystemPETSc<scalar>::_assembleMatrixIfNeeded()
{
if (_comm == PETSC_COMM_WORLD){
if (Msg::GetCommSize()>1){
int value = _valuesNotAssembled ? 1 : 0;
int sumValue = 0;
MPI_Allreduce((void*)&value, (void*)&sumValue, 1, MPI_INT, MPI_SUM, _comm);
if ((sumValue > 0) &&(sumValue < Msg::GetCommSize())){
_valuesNotAssembled= 1;
}
}
}
if (_valuesNotAssembled) {
_try(MatAssemblyBegin(_a, MAT_FINAL_ASSEMBLY));
_try(MatAssemblyEnd(_a, MAT_FINAL_ASSEMBLY));
_matrixChangedSinceLastSolve = true;
_valuesNotAssembled = false;
}
}
void linearSystemPETSc<fullMatrix<double> >::addToRightHandSide(int row,
const fullMatrix<double> &val)
{
if (!_entriesPreAllocated)
preAllocateEntries();
int blockSize;
_try(MatGetBlockSize(_a, &blockSize));
for (int ii = 0; ii < blockSize; ii++) {
PetscInt i = row * blockSize + ii;
PetscScalar v = val(ii, 0);
VecSetValues(_b, 1, &i, &v, ADD_VALUES);
}
}
void linearSystemPETSc<scalar>::print()
{
_assembleMatrixIfNeeded();
_try(VecAssemblyBegin(_b));
_try(VecAssemblyEnd(_b));
/*
PetscViewer fd;
_try(PetscViewerASCIIOpen(PETSC_COMM_WORLD, "mat.m", &fd));
_try(PetscViewerSetFormat(fd, PETSC_VIEWER_ASCII_MATLAB));
_try(PetscObjectSetName((PetscObject)_a, "A"));
_try(MatView(_a, fd));
*/
if(Msg::GetCommRank()==0)
printf("a :\n");
MatView(_a, PETSC_VIEWER_STDOUT_WORLD);
if(Msg::GetCommRank()==0)
printf("b :\n");
VecView(_b, PETSC_VIEWER_STDOUT_WORLD);
if(Msg::GetCommRank()==0)
printf("x :\n");
VecView(_x, PETSC_VIEWER_STDOUT_WORLD);
}
void linearSystemPETSc<fullMatrix<double> >::getFromSolution(int row, fullMatrix<double> &val) const
{
int blockSize;
_try(MatGetBlockSize(_a, &blockSize));
for (int i = 0; i < blockSize; i++) {
int ii = row*blockSize +i;
#ifdef PETSC_USE_COMPLEX
PetscScalar s;
VecGetValues ( _x, 1, &ii, &s);
val(i,0) = s.real();
#else
VecGetValues ( _x, 1, &ii, &val(i,0));
#endif
}
}
void linearSystemPETSc<scalar>::zeroMatrix()
{
if (_comm == PETSC_COMM_WORLD){
if (Msg::GetCommSize()>1){
int value = _entriesPreAllocated ? 1 : 0;
int sumValue = 0;
MPI_Allreduce((void*)&value, (void*)&sumValue, 1, MPI_INT, MPI_SUM, _comm);
if ((sumValue >= 0) &&(sumValue < Msg::GetCommSize()) && !_entriesPreAllocated){
preAllocateEntries();
}
}
}
if (_isAllocated && _entriesPreAllocated) {
_assembleMatrixIfNeeded();
_try(MatZeroEntries(_a));
}
}
void linearSystemPETSc<scalar>::_kspCreate()
{
// Set option given by the user in its (python script) without using argc,argv or .petscrc
if(this->_parameters.count("petsc_solver_options"))
_try(PetscOptionsInsertString(this->_parameters["petsc_solver_options"].c_str()));
_try(KSPCreate(_comm, &_ksp));
PC pc;
_try(KSPGetPC(_ksp, &pc));
// set some default options
//_try(PCSetType(pc, PCLU));//LU for direct solver and PCILU for indirect solver
/* _try(PCFactorSetMatOrderingType(pc, MATORDERING_RCM));
_try(PCFactorSetLevels(pc, 1));*/
_try(KSPSetTolerances(_ksp, 1.e-8, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT));
// override the default options with the ones from the option
// database (if any)
if (this->_parameters.count("petscPrefix"))
_try(KSPAppendOptionsPrefix(_ksp, this->_parameters["petscPrefix"].c_str()));
_try(KSPSetFromOptions(_ksp));
_try(PCSetFromOptions(pc));
_kspAllocated = true;
}
int linearSystemPETSc<scalar>::systemSolve()
{
if (!_kspAllocated)
_kspCreate();
_assembleMatrixIfNeeded();
if (!_matrixChangedSinceLastSolve || linearSystem<scalar>::_parameters["matrix_reuse"] == "same_matrix")
_try(KSPSetOperators(_ksp, _a, _a, SAME_PRECONDITIONER));
else if (linearSystem<scalar>::_parameters["matrix_reuse"] == "same_sparsity")
_try(KSPSetOperators(_ksp, _a, _a, SAME_NONZERO_PATTERN));
else
_try(KSPSetOperators(_ksp, _a, _a, DIFFERENT_NONZERO_PATTERN));
_matrixChangedSinceLastSolve = false;
/*MatInfo info;
MatGetInfo(_a, MAT_LOCAL, &info);
printf("mallocs %.0f nz_allocated %.0f nz_used %.0f nz_unneeded %.0f\n", info.mallocs, info.nz_allocated, info.nz_used, info.nz_unneeded);*/
_try(VecAssemblyBegin(_b));
_try(VecAssemblyEnd(_b));
_try(KSPSolve(_ksp, _b, _x));
//_try(KSPView(ksp, PETSC_VIEWER_STDOUT_SELF));
//PetscInt its;
//_try(KSPGetIterationNumber(ksp, &its));
//Msg::Info("%d iterations", its);
return 1;
}
linearSystemPETSc<scalar>::~linearSystemPETSc()
{
clear();
if(_kspAllocated)
_try(KSPDestroy(&_ksp));
}
int linearSystemPETSc<scalar>::getNumKspIteration() const
{
int n;
_try(KSPGetIterationNumber(_ksp, &n));
return n;
}
bool eigenSolver::solve(int numEigenValues, std::string which)
{
if(!_A) return false;
Mat A = _A->getMatrix();
Mat B = _B ? _B->getMatrix() : PETSC_NULL;
PetscInt N, M;
_try(MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY));
_try(MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY));
_try(MatGetSize(A, &N, &M));
PetscInt N2, M2;
if (_B) {
_try(MatAssemblyBegin(B, MAT_FINAL_ASSEMBLY));
_try(MatAssemblyEnd(B, MAT_FINAL_ASSEMBLY));
_try(MatGetSize(B, &N2, &M2));
}
// generalized eigenvalue problem A x - \lambda B x = 0
EPS eps;
_try(EPSCreate(PETSC_COMM_WORLD, &eps));
_try(EPSSetOperators(eps, A, B));
if(_hermitian)
_try(EPSSetProblemType(eps, _B ? EPS_GHEP : EPS_HEP));
else
_try(EPSSetProblemType(eps, _B ? EPS_GNHEP : EPS_NHEP));
// set some default options
_try(EPSSetDimensions(eps, numEigenValues, PETSC_DECIDE, PETSC_DECIDE));
_try(EPSSetTolerances(eps, 1.e-7, 20));//1.e-7 20
_try(EPSSetType(eps, EPSKRYLOVSCHUR)); //default
//_try(EPSSetType(eps, EPSARNOLDI));
//_try(EPSSetType(eps, EPSARPACK));
//_try(EPSSetType(eps, EPSPOWER));
// override these options at runtime, petsc-style
_try(EPSSetFromOptions(eps));
// force options specified directly as arguments
if(numEigenValues)
_try(EPSSetDimensions(eps, numEigenValues, PETSC_DECIDE, PETSC_DECIDE));
if(which == "smallest")
_try(EPSSetWhichEigenpairs(eps, EPS_SMALLEST_MAGNITUDE));
else if(which == "smallestReal")
_try(EPSSetWhichEigenpairs(eps, EPS_SMALLEST_REAL));
else if(which == "largest")
_try(EPSSetWhichEigenpairs(eps, EPS_LARGEST_MAGNITUDE));
// print info
#if (SLEPC_VERSION_RELEASE == 0 || (SLEPC_VERSION_MAJOR > 3 || (SLEPC_VERSION_MAJOR == 3 && SLEPC_VERSION_MINOR >= 4)))
EPSType type;
#else
const EPSType type;
#endif
_try(EPSGetType(eps, &type));
Msg::Debug("SLEPc solution method: %s", type);
PetscInt nev;
_try(EPSGetDimensions(eps, &nev, PETSC_NULL, PETSC_NULL));
Msg::Debug("SLEPc number of requested eigenvalues: %d", nev);
PetscReal tol;
PetscInt maxit;
_try(EPSGetTolerances(eps, &tol, &maxit));
Msg::Debug("SLEPc stopping condition: tol=%g, maxit=%d", tol, maxit);
// solve
Msg::Info("SLEPc solving...");
double t1 = Cpu();
_try(EPSSolve(eps));
// check convergence
int its;
_try(EPSGetIterationNumber(eps, &its));
EPSConvergedReason reason;
_try(EPSGetConvergedReason(eps, &reason));
if(reason == EPS_CONVERGED_TOL){
double t2 = Cpu();
Msg::Debug("SLEPc converged in %d iterations (%g s)", its, t2-t1);
}
else if(reason == EPS_DIVERGED_ITS)
Msg::Error("SLEPc diverged after %d iterations", its);
else if(reason == EPS_DIVERGED_BREAKDOWN)
Msg::Error("SLEPc generic breakdown in method");
#if (SLEPC_VERSION_MAJOR < 3 || (SLEPC_VERSION_MAJOR == 3 && SLEPC_VERSION_MINOR < 2))
else if(reason == EPS_DIVERGED_NONSYMMETRIC)
Msg::Error("The operator is nonsymmetric");
#endif
// get number of converged approximate eigenpairs
PetscInt nconv;
_try(EPSGetConverged(eps, &nconv));
Msg::Debug("SLEPc number of converged eigenpairs: %d", nconv);
// ignore additional eigenvalues if we get more than what we asked
if(nconv > nev) nconv = nev;
if (nconv > 0) {
Vec xr, xi;
_try(MatGetVecs(A, PETSC_NULL, &xr));
_try(MatGetVecs(A, PETSC_NULL, &xi));
Msg::Debug(" Re[EigenValue] Im[EigenValue]"
" Relative error");
for (int i = 0; i < nconv; i++){
PetscScalar kr, ki;
_try(EPSGetEigenpair(eps, i, &kr, &ki, xr, xi));
PetscReal error;
_try(EPSComputeRelativeError(eps, i, &error));
#if defined(PETSC_USE_COMPLEX)
PetscReal re = PetscRealPart(kr);
PetscReal im = PetscImaginaryPart(kr);
#else
PetscReal re = kr;
PetscReal im = ki;
#endif
Msg::Debug("EIG %03d %s%.16e %s%.16e %3.6e",
i, (re < 0) ? "" : " ", re, (im < 0) ? "" : " ", im, error);
// store eigenvalues and eigenvectors
_eigenValues.push_back(std::complex<double>(re, im));
PetscScalar *tmpr, *tmpi;
_try(VecGetArray(xr, &tmpr));
_try(VecGetArray(xi, &tmpi));
std::vector<std::complex<double> > ev(N);
for(int i = 0; i < N; i++){
#if defined(PETSC_USE_COMPLEX)
ev[i] = tmpr[i];
#else
ev[i] = std::complex<double>(tmpr[i], tmpi[i]);
#endif
}
_eigenVectors.push_back(ev);
}
_try(VecDestroy(&xr));
_try(VecDestroy(&xi));
}
_try(EPSDestroy(&eps));
if(reason == EPS_CONVERGED_TOL){
Msg::Debug("SLEPc done");
return true;
}
else{
Msg::Warning("SLEPc failed");
return false;
}
}
void linearSystemPETSc<scalar>::allocate(int nbRows)
{
int commSize;
MPI_Comm_size(_comm, &commSize);
int blockSize = _getBlockSizeFromParameters();
clear();
_try(MatCreate(_comm, &_a));
_try(MatSetSizes(_a, blockSize * nbRows, blockSize * nbRows, PETSC_DETERMINE, PETSC_DETERMINE));
if (blockSize > 1) {
if (commSize > 1) {
_try(MatSetType(_a, MATMPIBAIJ));
}
else {
_try(MatSetType(_a, MATSEQBAIJ));
}
}
// override the default options with the ones from the option
// database (if any)
if (this->_parameters.count("petscOptions"))
_try(PetscOptionsInsertString(this->_parameters["petscOptions"].c_str()));
if (this->_parameters.count("petscPrefix"))
_try(MatAppendOptionsPrefix(_a, this->_parameters["petscPrefix"].c_str()));
_try(MatSetFromOptions(_a));
//since PETSc 3.3 GetOwnershipRange and MatGetSize cannot be called before MatXXXSetPreallocation
_localSize = nbRows;
#ifdef HAVE_MPI
if (commSize>1){
_localRowStart = 0;
if (Msg::GetCommRank() != 0) {
MPI_Status status;
MPI_Recv((void*)&_localRowStart, 1, MPI_INT, Msg::GetCommRank() - 1, 1, MPI_COMM_WORLD, &status);
}
_localRowEnd = _localRowStart + nbRows;
if (Msg::GetCommRank() != Msg::GetCommSize() - 1) {
MPI_Send((void*)&_localRowEnd, 1, MPI_INT, Msg::GetCommRank() + 1, 1, MPI_COMM_WORLD);
}
MPI_Allreduce((void*)&_localSize, (void*)&_globalSize, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
}
else{
_localRowStart = 0;
_localRowEnd = nbRows;
_globalSize = _localSize;
}
#else
_localRowStart = 0;
_localRowEnd = nbRows;
_globalSize = _localSize;
#endif
// preallocation option must be set after other options
_try(VecCreate(_comm, &_x));
_try(VecSetSizes(_x, blockSize * nbRows, PETSC_DETERMINE));
// override the default options with the ones from the option
// database (if any)
if (this->_parameters.count("petscPrefix"))
_try(VecAppendOptionsPrefix(_x, this->_parameters["petscPrefix"].c_str()));
_try(VecSetFromOptions(_x));
_try(VecDuplicate(_x, &_b));
_isAllocated = true;
_entriesPreAllocated = false;
}
void linearSystemPETSc<scalar>::addToSolution(int row, const scalar &val)
{
PetscInt i = row;
PetscScalar s = val;
_try(VecSetValues(_x, 1, &i, &s, ADD_VALUES));
}
void linearSystemPETSc<scalar>::addToRightHandSide(int row, const scalar &val)
{
PetscInt i = row;
PetscScalar s = val;
_try(VecSetValues(_b, 1, &i, &s, ADD_VALUES));
}
