void PETScLinearSolver::UpdateSolutions(PetscScalar *u)
{
#ifdef TEST_MEM_PETSC
//TEST
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
PetscScalar *xp = NULL; //nullptr;
VecGetArray(x, &xp);
gatherLocalVectors(xp, u);
//MPI_Barrier(PETSC_COMM_WORLD);
VecRestoreArray(x, &xp);
//TEST
#ifdef TEST_MEM_PETSC
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(PETSC_COMM_WORLD, "### Memory usage by Updating. Before :%f After:%f Increase:%d\n", mem1, mem2, (int)(mem2 - mem1));
#endif
}
void PETScVector::getGlobalVector(PetscScalar u[])
{
#ifdef TEST_MEM_PETSC
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
PetscScalar *xp = nullptr;
VecGetArray(_v, &xp);
gatherLocalVectors(xp, u);
//This following line may be needed late on
// for a communication load balance:
//MPI_Barrier(PETSC_COMM_WORLD);
VecRestoreArray(_v, &xp);
//TEST
#ifdef TEST_MEM_PETSC
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(PETSC_COMM_WORLD, "### Memory usage by Updating. Before :%f After:%f Increase:%d\n", mem1, mem2, (int)(mem2 - mem1));
#endif
}
/*@C
PetscTraceBackErrorHandler - Default error handler routine that generates
a traceback on error detection.
Not Collective
Input Parameters:
+ comm - communicator over which error occurred
. line - the line number of the error (indicated by __LINE__)
. func - the function where error is detected (indicated by __FUNCT__)
. file - the file in which the error was detected (indicated by __FILE__)
. mess - an error text string, usually just printed to the screen
. n - the generic error number
. p - PETSC_ERROR_INITIAL if this is the first call the error handler, otherwise PETSC_ERROR_REPEAT
- ctx - error handler context
Level: developer
Notes:
Most users need not directly employ this routine and the other error
handlers, but can instead use the simplified interface SETERRQ, which has
the calling sequence
$ SETERRQ(comm,number,n,mess)
Notes for experienced users:
Use PetscPushErrorHandler() to set the desired error handler. The
currently available PETSc error handlers include PetscTraceBackErrorHandler(),
PetscAttachDebuggerErrorHandler(), PetscAbortErrorHandler(), and PetscMPIAbortErrorHandler()
Concepts: error handler^traceback
Concepts: traceback^generating
.seealso: PetscPushErrorHandler(), PetscAttachDebuggerErrorHandler(),
PetscAbortErrorHandler()
@*/
PetscErrorCode PetscTraceBackErrorHandler(MPI_Comm comm,int line,const char *fun,const char *file,PetscErrorCode n,PetscErrorType p,const char *mess,void *ctx)
{
PetscLogDouble mem,rss;
PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE;
PetscMPIInt rank = 0;
PetscFunctionBegin;
if (comm != PETSC_COMM_SELF) MPI_Comm_rank(comm,&rank);
if (!rank) {
PetscBool ismain,isunknown;
static int cnt = 1;
if (p == PETSC_ERROR_INITIAL) {
PetscErrorPrintfHilight();
(*PetscErrorPrintf)("--------------------- Error Message --------------------------------------------------------------\n");
PetscErrorPrintfNormal();
if (n == PETSC_ERR_MEM) {
(*PetscErrorPrintf)("Out of memory. This could be due to allocating\n");
(*PetscErrorPrintf)("too large an object or bleeding by not properly\n");
(*PetscErrorPrintf)("destroying unneeded objects.\n");
PetscMallocGetCurrentUsage(&mem);
PetscMemoryGetCurrentUsage(&rss);
PetscOptionsGetBool(NULL,"-malloc_dump",&flg1,NULL);
PetscOptionsGetBool(NULL,"-malloc_log",&flg2,NULL);
PetscOptionsHasName(NULL,"-malloc_log_threshold",&flg3);
if (flg2 || flg3) PetscMallocDumpLog(stdout);
else {
(*PetscErrorPrintf)("Memory allocated %.0f Memory used by process %.0f\n",mem,rss);
if (flg1) PetscMallocDump(stdout);
else (*PetscErrorPrintf)("Try running with -malloc_dump or -malloc_log for info.\n");
}
} else {
const char *text;
PetscErrorMessage(n,&text,NULL);
if (text) (*PetscErrorPrintf)("%s\n",text);
}
if (mess) (*PetscErrorPrintf)("%s\n",mess);
(*PetscErrorPrintf)("See http://www.mcs.anl.gov/petsc/documentation/faq.html for trouble shooting.\n");
(*PetscErrorPrintf)("%s\n",version);
if (PetscErrorPrintfInitializeCalled) (*PetscErrorPrintf)("%s on a %s named %s by %s %s\n",pname,arch,hostname,username,date);
(*PetscErrorPrintf)("Configure options %s\n",petscconfigureoptions);
}
/* print line of stack trace */
(*PetscErrorPrintf)("#%d %s() line %d in %s\n",cnt++,fun,line,file);
PetscStrncmp(fun,"main",4,&ismain);
PetscStrncmp(fun,"unknown",7,&isunknown);
if (ismain || isunknown) {
PetscOptionsViewError();
PetscErrorPrintfHilight();
(*PetscErrorPrintf)("----------------End of Error Message -------send entire error message to [email protected]\n");
PetscErrorPrintfNormal();
}
} else {
/* do not print error messages since process 0 will print them, sleep before aborting so will not accidently kill process 0*/
PetscSleep(10.0);
abort();
}
PetscFunctionReturn(n);
}
/*@C
PetscMemoryShowUsage - Shows the amount of memory currently being used
in a communicator.
Collective on PetscViewer
Input Parameter:
+ viewer - the viewer that defines the communicator
- message - string printed before values
Level: intermediate
Concepts: memory usage
.seealso: PetscMallocDump(), PetscMemoryGetCurrentUsage()
@*/
PetscErrorCode PetscMemoryShowUsage(PetscViewer viewer,const char message[])
{
PetscLogDouble allocated,maximum,resident,residentmax;
PetscErrorCode ierr;
PetscMPIInt rank;
MPI_Comm comm;
PetscFunctionBegin;
if (!viewer) viewer = PETSC_VIEWER_STDOUT_WORLD;
ierr = PetscMallocGetCurrentUsage(&allocated);CHKERRQ(ierr);
ierr = PetscMallocGetMaximumUsage(&maximum);CHKERRQ(ierr);
ierr = PetscMemoryGetCurrentUsage(&resident);CHKERRQ(ierr);
ierr = PetscMemoryGetMaximumUsage(&residentmax);CHKERRQ(ierr);
if (residentmax > 0) residentmax = PetscMax(resident,residentmax);
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,message);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
if (resident && residentmax && allocated) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"[%d]Current space PetscMalloc()ed %g, max space PetscMalloced() %g\n[%d]Current process memory %g max process memory %g\n",rank,allocated,maximum,rank,resident,residentmax);CHKERRQ(ierr);
} else if (resident && residentmax) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"[%d]Run with -malloc to get statistics on PetscMalloc() calls\n[%d]Current process memory %g max process memory %g\n",rank,rank,resident,residentmax);CHKERRQ(ierr);
} else if (resident && allocated) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"[%d]Current space PetscMalloc()ed %g, max space PetscMalloced() %g\n[%d]Current process memory %g, run with -memory_info to get max memory usage\n",rank,allocated,maximum,rank,resident);CHKERRQ(ierr);
} else if (allocated) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"[%d]Current space PetscMalloc()ed %g, max space PetscMalloced() %g\n[%d]OS cannot compute process memory\n",rank,allocated,maximum,rank);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"Run with -malloc to get statistics on PetscMalloc() calls\nOS cannot compute process memory\n");CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
PetscHeaderDestroy_Private - Destroys a base PETSc object header. Called by
the macro PetscHeaderDestroy().
*/
PetscErrorCode PetscHeaderDestroy_Private(PetscObject h)
{
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeader(h,1);
ierr = PetscLogObjectDestroy(h);CHKERRQ(ierr);
ierr = PetscComposedQuantitiesDestroy(h);CHKERRQ(ierr);
if (PetscMemoryCollectMaximumUsage) {
PetscLogDouble usage;
ierr = PetscMemoryGetCurrentUsage(&usage);CHKERRQ(ierr);
if (usage > PetscMemoryMaximumUsage) PetscMemoryMaximumUsage = usage;
}
/* first destroy things that could execute arbitrary code */
if (h->python_destroy) {
void *python_context = h->python_context;
PetscErrorCode (*python_destroy)(void*) = h->python_destroy;
h->python_context = 0;
h->python_destroy = 0;
ierr = (*python_destroy)(python_context);CHKERRQ(ierr);
}
ierr = PetscObjectDestroyOptionsHandlers(h);CHKERRQ(ierr);
ierr = PetscObjectListDestroy(&h->olist);CHKERRQ(ierr);
ierr = PetscCommDestroy(&h->comm);CHKERRQ(ierr);
/* next destroy other things */
h->classid = PETSCFREEDHEADER;
ierr = PetscFunctionListDestroy(&h->qlist);CHKERRQ(ierr);
ierr = PetscFree(h->type_name);CHKERRQ(ierr);
ierr = PetscFree(h->name);CHKERRQ(ierr);
ierr = PetscFree(h->prefix);CHKERRQ(ierr);
ierr = PetscFree(h->fortran_func_pointers);CHKERRQ(ierr);
ierr = PetscFree(h->fortrancallback[PETSC_FORTRAN_CALLBACK_CLASS]);CHKERRQ(ierr);
ierr = PetscFree(h->fortrancallback[PETSC_FORTRAN_CALLBACK_SUBTYPE]);CHKERRQ(ierr);
#if defined(PETSC_USE_LOG)
if (PetscObjectsLog) {
PetscInt i;
/* Record object removal from list of all objects */
for (i=0; i<PetscObjectsMaxCounts; i++) {
if (PetscObjects[i] == h) {
PetscObjects[i] = 0;
PetscObjectsCounts--;
break;
}
}
if (!PetscObjectsCounts) {
ierr = PetscFree(PetscObjects);CHKERRQ(ierr);
PetscObjectsMaxCounts = 0;
}
}
#endif
PetscFunctionReturn(0);
}
inline PetscErrorCode ScatSMatMultFFT(Mat mat,Vec xx,Vec yy)
{
PetscInt N;
Vec FFTVec;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = VecGetSize(xx,&N);CHKERRQ(ierr);
//Convert to use 3D convolution theorem:
ierr = VecSet(RealSpaceCube,0);CHKERRQ(ierr);
ierr = VecCopyN1(xx,RealSpaceCube,N);CHKERRQ(ierr);
// Apply FFTW_FORWARD for xx:
fftwf_execute(fplan);
ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr);PetscPrintf(PETSC_COMM_WORLD,"Apply FFT:\t\tMem used: %G \t",mem);GetTime(0);
ierr = VecDuplicate(RealSpaceCube,&FFTVec);CHKERRQ(ierr);
ierr = VecCopy(RealSpaceCube,FFTVec);CHKERRQ(ierr);
//Prepare the matrix mat to multiply with xx, mat*xx=yy:
ierr = FFTPaddedGreenCube(dim);CHKERRQ(ierr);
ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr);PetscPrintf(PETSC_COMM_WORLD,"Pad Green cube:\t\tMem used: %G \t",mem);GetTime(0);
//Matrix-vector multiplication in Fourier space:
ierr = VecPointwiseMult(RealSpaceCube,RealSpaceCube,FFTVec);CHKERRQ(ierr);
ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr);PetscPrintf(PETSC_COMM_WORLD,"Multiply vectors:\tMem used: %G \t",mem);GetTime(0);
ierr = VecDestroy(&FFTVec);CHKERRQ(ierr);
// Apply FFTW_BACKWARD:
fftwf_execute(bplan);
ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr);PetscPrintf(PETSC_COMM_WORLD,"Apply iFFT:\t\tMem used: %G \t",mem);GetTime(0);
ierr = VecCopyN1(RealSpaceCube,yy,N);CHKERRQ(ierr);
ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr);PetscPrintf(PETSC_COMM_WORLD,"Assembly result:\tMem used: %G \t",mem);GetTime(0);
PetscFunctionReturn(0);
}
PetscErrorCode PetscMemoryTrace(const char label[])
{
PetscErrorCode ierr;
PetscLogDouble mem,mal;
static PetscLogDouble oldmem = 0,oldmal = 0;
PetscFunctionBegin;
ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr);
ierr = PetscMallocGetCurrentUsage(&mal);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);CHKERRQ(ierr);
oldmem = mem;
oldmal = mal;
PetscFunctionReturn(0);
}
/*
PetscHeaderDestroy_Private - Destroys a base PETSc object header. Called by
the macro PetscHeaderDestroy().
*/
PetscErrorCode PETSC_DLLEXPORT PetscHeaderDestroy_Private(PetscObject h)
{
PetscErrorCode ierr;
PetscFunctionBegin;
if (PetscMemoryCollectMaximumUsage) {
PetscLogDouble usage;
ierr = PetscMemoryGetCurrentUsage(&usage);CHKERRQ(ierr);
if (usage > PetscMemoryMaximumUsage) PetscMemoryMaximumUsage = usage;
}
/* first destroy things that could execute arbitrary code */
if (h->python_destroy) {
void *python_context = h->python_context;
PetscErrorCode (*python_destroy)(void*) = h->python_destroy;
h->python_context = 0;
h->python_destroy = 0;
ierr = (*python_destroy)(python_context);CHKERRQ(ierr);
}
ierr = PetscOListDestroy(h->olist);CHKERRQ(ierr);
ierr = PetscCommDestroy(&h->comm);CHKERRQ(ierr);
/* next destroy other things */
h->cookie = PETSCFREEDHEADER;
ierr = PetscFree(h->bops);CHKERRQ(ierr);
ierr = PetscFListDestroy(&h->qlist);CHKERRQ(ierr);
ierr = PetscStrfree(h->type_name);CHKERRQ(ierr);
ierr = PetscStrfree(h->name);CHKERRQ(ierr);
ierr = PetscStrfree(h->prefix);CHKERRQ(ierr);
ierr = PetscFree(h->fortran_func_pointers);CHKERRQ(ierr);
ierr = PetscFree(h->intcomposeddata);CHKERRQ(ierr);
ierr = PetscFree(h->intcomposedstate);CHKERRQ(ierr);
ierr = PetscFree(h->realcomposeddata);CHKERRQ(ierr);
ierr = PetscFree(h->realcomposedstate);CHKERRQ(ierr);
ierr = PetscFree(h->scalarcomposeddata);CHKERRQ(ierr);
ierr = PetscFree(h->scalarcomposedstate);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscTraceBackErrorHandler - Default error handler routine that generates
a traceback on error detection.
Not Collective
Input Parameters:
+ comm - communicator over which error occurred
. line - the line number of the error (indicated by __LINE__)
. func - the function where error is detected (indicated by __FUNCT__)
. file - the file in which the error was detected (indicated by __FILE__)
. dir - the directory of the file (indicated by __SDIR__)
. mess - an error text string, usually just printed to the screen
. n - the generic error number
. p - PETSC_ERROR_INITIAL if this is the first call the the error handler, otherwise PETSC_ERROR_REPEAT
- ctx - error handler context
Level: developer
Notes:
Most users need not directly employ this routine and the other error
handlers, but can instead use the simplified interface SETERRQ, which has
the calling sequence
$ SETERRQ(comm,number,n,mess)
Notes for experienced users:
Use PetscPushErrorHandler() to set the desired error handler. The
currently available PETSc error handlers include PetscTraceBackErrorHandler(),
PetscAttachDebuggerErrorHandler(), PetscAbortErrorHandler(), and PetscMPIAbortErrorHandler()
Concepts: error handler^traceback
Concepts: traceback^generating
.seealso: PetscPushErrorHandler(), PetscAttachDebuggerErrorHandler(),
PetscAbortErrorHandler()
@*/
PetscErrorCode PetscTraceBackErrorHandler(MPI_Comm comm,int line,const char *fun,const char* file,const char *dir,PetscErrorCode n,PetscErrorType p,const char *mess,void *ctx)
{
PetscLogDouble mem,rss;
PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE;
PetscMPIInt rank = 0;
PetscFunctionBegin;
if (comm != PETSC_COMM_SELF) {
MPI_Comm_rank(comm,&rank);
}
if (!rank) {
if (p == PETSC_ERROR_INITIAL) {
(*PetscErrorPrintf)("--------------------- Error Message ------------------------------------\n");
if (n == PETSC_ERR_MEM) {
(*PetscErrorPrintf)("Out of memory. This could be due to allocating\n");
(*PetscErrorPrintf)("too large an object or bleeding by not properly\n");
(*PetscErrorPrintf)("destroying unneeded objects.\n");
PetscMallocGetCurrentUsage(&mem);
PetscMemoryGetCurrentUsage(&rss);
PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);
PetscOptionsGetBool(PETSC_NULL,"-malloc_log",&flg2,PETSC_NULL);
PetscOptionsHasName(PETSC_NULL,"-malloc_log_threshold",&flg3);
if (flg2 || flg3) {
PetscMallocDumpLog(stdout);
} else {
(*PetscErrorPrintf)("Memory allocated %.0f Memory used by process %.0f\n",mem,rss);
if (flg1) {
PetscMallocDump(stdout);
} else {
(*PetscErrorPrintf)("Try running with -malloc_dump or -malloc_log for info.\n");
}
}
} else {
const char *text;
PetscErrorMessage(n,&text,PETSC_NULL);
if (text) (*PetscErrorPrintf)("%s!\n",text);
}
if (mess) {
(*PetscErrorPrintf)("%s!\n",mess);
}
(*PetscErrorPrintf)("------------------------------------------------------------------------\n");
(*PetscErrorPrintf)("%s\n",version);
(*PetscErrorPrintf)("See docs/changes/index.html for recent updates.\n");
(*PetscErrorPrintf)("See docs/faq.html for hints about trouble shooting.\n");
(*PetscErrorPrintf)("See docs/index.html for manual pages.\n");
(*PetscErrorPrintf)("------------------------------------------------------------------------\n");
if (PetscErrorPrintfInitializeCalled) {
(*PetscErrorPrintf)("%s on a %s named %s by %s %s\n",pname,arch,hostname,username,date);
}
(*PetscErrorPrintf)("Libraries linked from %s\n",PETSC_LIB_DIR);
(*PetscErrorPrintf)("Configure run at %s\n",petscconfigureruntime);
(*PetscErrorPrintf)("Configure options %s\n",petscconfigureoptions);
(*PetscErrorPrintf)("------------------------------------------------------------------------\n");
}
/* print line of stack trace */
(*PetscErrorPrintf)("%s() line %d in %s%s\n",fun,line,dir,file);
} else {
/* do not print error messages since process 0 will print them, sleep before aborting so will not accidently kill process 0*/
PetscSleep(10.0);
abort();
}
PetscFunctionReturn(n);
}
void PETScLinearSolver::Solver()
{
//TEST
#ifdef TEST_MEM_PETSC
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
/*
//TEST
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "x.txt", &viewer);
PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
PetscObjectSetName((PetscObject)x,"Solution");
VecView(x, viewer);
*/
int its;
PetscLogDouble v1,v2;
KSPConvergedReason reason;
// #define PETSC34
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v1);
#else
PetscGetTime(&v1);
#endif
#if (PETSC_VERSION_MAJOR == 3) && (PETSC_VERSION_MINOR > 4)
KSPSetOperators(lsolver, A, A);
#else
KSPSetOperators(lsolver, A, A, DIFFERENT_NONZERO_PATTERN);
#endif
KSPSolve(lsolver, b, x);
KSPGetConvergedReason(lsolver,&reason); //CHKERRQ(ierr);
if (reason==KSP_DIVERGED_INDEFINITE_PC)
{
PetscPrintf(PETSC_COMM_WORLD,"\nDivergence because of indefinite preconditioner;\n");
PetscPrintf(PETSC_COMM_WORLD,"Run the executable again but with -pc_factor_shift_positive_definite option.\n");
}
else if (reason<0)
{
PetscPrintf(PETSC_COMM_WORLD,"\nOther kind of divergence: this should not happen.\n");
}
else
{
const char *slv_type;
const char *prc_type;
KSPGetType(lsolver, &slv_type);
PCGetType(prec, &prc_type);
PetscPrintf(PETSC_COMM_WORLD,"\n================================================");
PetscPrintf(PETSC_COMM_WORLD, "\nLinear solver %s with %s preconditioner",
slv_type, prc_type);
KSPGetIterationNumber(lsolver,&its); //CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"\nConvergence in %d iterations.\n",(int)its);
PetscPrintf(PETSC_COMM_WORLD,"\n================================================");
}
PetscPrintf(PETSC_COMM_WORLD,"\n");
//VecAssemblyBegin(x);
//VecAssemblyEnd(x);
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v2);
#else
PetscGetTime(&v2);
#endif
time_elapsed += v2-v1;
#define aTEST_OUT
#ifdef TEST_OUT
//TEST
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "x2.txt", &viewer);
PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
PetscObjectSetName((PetscObject)A,"Matrix");
MatView(A, viewer);
PetscObjectSetName((PetscObject)x,"Solution");
VecView(x, viewer);
PetscObjectSetName((PetscObject)b,"RHS");
VecView(b, viewer);
VecDestroy(&b);
VecDestroy(&x);
MatDestroy(&A);
if(lsolver) KSPDestroy(&lsolver);
// if(prec) PCDestroy(&prec);
if(global_x0)
delete [] global_x0;
if(global_x1)
delete [] global_x1;
PetscFinalize();
exit(0);
#endif
#ifdef TEST_MEM_PETSC
//TEST
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(PETSC_COMM_WORLD, "###Memory usage by solver. Before :%f After:%f Increase:%d\n", mem1, mem2, (int)(mem2 - mem1));
#endif
}
PETSC_EXTERN void PETSC_STDCALL petscmemorygetcurrentusage_(PetscLogDouble *mem, int *__ierr ){
*__ierr = PetscMemoryGetCurrentUsage(mem);
}
bool PETScLinearSolver::solve(PETScMatrix& A, PETScVector& b, PETScVector& x)
{
BaseLib::RunTime wtimer;
wtimer.start();
// define TEST_MEM_PETSC
#ifdef TEST_MEM_PETSC
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
#if (PETSC_VERSION_NUMBER > 3040)
KSPSetOperators(_solver, A.getRawMatrix(), A.getRawMatrix());
#else
KSPSetOperators(_solver, A.getRawMatrix(), A.getRawMatrix(),
DIFFERENT_NONZERO_PATTERN);
#endif
KSPSolve(_solver, b.getRawVector(), x.getRawVector());
KSPConvergedReason reason;
KSPGetConvergedReason(_solver, &reason);
bool converged = true;
if (reason > 0)
{
const char* ksp_type;
const char* pc_type;
KSPGetType(_solver, &ksp_type);
PCGetType(_pc, &pc_type);
PetscPrintf(PETSC_COMM_WORLD,
"\n================================================");
PetscPrintf(PETSC_COMM_WORLD,
"\nLinear solver %s with %s preconditioner", ksp_type,
pc_type);
PetscInt its;
KSPGetIterationNumber(_solver, &its);
PetscPrintf(PETSC_COMM_WORLD, "\nconverged in %d iterations", its);
switch (reason)
{
case KSP_CONVERGED_RTOL:
PetscPrintf(PETSC_COMM_WORLD,
" (relative convergence criterion fulfilled).");
break;
case KSP_CONVERGED_ATOL:
PetscPrintf(PETSC_COMM_WORLD,
" (absolute convergence criterion fulfilled).");
break;
default:
PetscPrintf(PETSC_COMM_WORLD, ".");
}
PetscPrintf(PETSC_COMM_WORLD,
"\n================================================\n");
}
else if (reason == KSP_DIVERGED_ITS)
{
const char* ksp_type;
const char* pc_type;
KSPGetType(_solver, &ksp_type);
PCGetType(_pc, &pc_type);
PetscPrintf(PETSC_COMM_WORLD,
"\nLinear solver %s with %s preconditioner", ksp_type,
pc_type);
PetscPrintf(PETSC_COMM_WORLD,
"\nWarning: maximum number of iterations reached.\n");
}
else
{
converged = false;
if (reason == KSP_DIVERGED_INDEFINITE_PC)
{
PetscPrintf(PETSC_COMM_WORLD,
"\nDivergence because of indefinite preconditioner,");
PetscPrintf(PETSC_COMM_WORLD,
"\nTry to run again with "
"-pc_factor_shift_positive_definite option.\n");
}
else if (reason == KSP_DIVERGED_BREAKDOWN_BICG)
{
PetscPrintf(PETSC_COMM_WORLD,
"\nKSPBICG method was detected so the method could not "
"continue to enlarge the Krylov space.");
PetscPrintf(PETSC_COMM_WORLD,
"\nTry to run again with another solver.\n");
}
else if (reason == KSP_DIVERGED_NONSYMMETRIC)
{
PetscPrintf(PETSC_COMM_WORLD,
"\nMatrix or preconditioner is unsymmetric but KSP "
"requires symmetric.\n");
}
else
{
PetscPrintf(PETSC_COMM_WORLD,
"\nDivergence detected, use command option "
"-ksp_monitor or -log_summary to check the details.\n");
}
}
#ifdef TEST_MEM_PETSC
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(
PETSC_COMM_WORLD,
"###Memory usage by solver. Before: %f After: %f Increase: %d\n", mem1,
mem2, (int)(mem2 - mem1));
#endif
_elapsed_ctime += wtimer.elapsed();
return converged;
}
/*@C
PetscMemoryView - Shows the amount of memory currently being used
in a communicator.
Collective on PetscViewer
Input Parameter:
+ viewer - the viewer that defines the communicator
- message - string printed before values
Options Database:
+ -malloc - have PETSc track how much memory it has allocated
- -memory_view - during PetscFinalize() have this routine called
Level: intermediate
Concepts: memory usage
.seealso: PetscMallocDump(), PetscMemoryGetCurrentUsage(), PetscMemorySetGetMaximumUsage()
@*/
PetscErrorCode PetscMemoryView(PetscViewer viewer,const char message[])
{
PetscLogDouble allocated,allocatedmax,resident,residentmax,gallocated,gallocatedmax,gresident,gresidentmax,maxgallocated,maxgallocatedmax,maxgresident,maxgresidentmax;
PetscLogDouble mingallocated,mingallocatedmax,mingresident,mingresidentmax;
PetscErrorCode ierr;
MPI_Comm comm;
PetscFunctionBegin;
if (!viewer) viewer = PETSC_VIEWER_STDOUT_WORLD;
ierr = PetscMallocGetCurrentUsage(&allocated);CHKERRQ(ierr);
ierr = PetscMallocGetMaximumUsage(&allocatedmax);CHKERRQ(ierr);
ierr = PetscMemoryGetCurrentUsage(&resident);CHKERRQ(ierr);
ierr = PetscMemoryGetMaximumUsage(&residentmax);CHKERRQ(ierr);
if (residentmax > 0) residentmax = PetscMax(resident,residentmax);
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,message);CHKERRQ(ierr);
if (resident && residentmax && allocated) {
ierr = MPI_Reduce(&residentmax,&gresidentmax,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&residentmax,&maxgresidentmax,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&residentmax,&mingresidentmax,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Maximum (over computational time) process memory: total %5.4e max %5.4e min %5.4e\n",gresidentmax,maxgresidentmax,mingresidentmax);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&gresident,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&maxgresident,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&mingresident,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Current process memory: total %5.4e max %5.4e min %5.4e\n",gresident,maxgresident,mingresident);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocatedmax,&gallocatedmax,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocatedmax,&maxgallocatedmax,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocatedmax,&mingallocatedmax,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Maximum (over computational time) space PetscMalloc()ed: total %5.4e max %5.4e min %5.4e\n",gallocatedmax,maxgallocatedmax,mingallocatedmax);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&gallocated,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&maxgallocated,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&mingallocated,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Current space PetscMalloc()ed: total %5.4e max %5.4e min %5.4e\n",gallocated,maxgallocated,mingallocated);CHKERRQ(ierr);
} else if (resident && residentmax) {
ierr = MPI_Reduce(&residentmax,&gresidentmax,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&residentmax,&maxgresidentmax,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&residentmax,&mingresidentmax,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Maximum (over computational time) process memory: total %5.4e max %5.4e min %5.4e\n",gresidentmax,maxgresidentmax,mingresidentmax);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&gresident,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&maxgresident,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&mingresident,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Current process memory: total %5.4e max %5.4e min %5.4e\n",gresident,maxgresident,mingresident);CHKERRQ(ierr);
} else if (resident && allocated) {
ierr = MPI_Reduce(&resident,&gresident,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&maxgresident,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&resident,&mingresident,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Current process memory: total %5.4e max %5.4e min %5.4e\n",gresident,maxgresident,mingresident);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&gallocated,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&maxgallocated,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&mingallocated,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Current space PetscMalloc()ed: total %5.4e max %5.4e min %5.4e\n",gallocated,maxgallocated,mingallocated);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Run with -memory_view to get maximum memory usage\n");CHKERRQ(ierr);
} else if (allocated) {
ierr = MPI_Reduce(&allocated,&gallocated,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&maxgallocated,1,MPIU_PETSCLOGDOUBLE,MPI_MAX,0,comm);CHKERRQ(ierr);
ierr = MPI_Reduce(&allocated,&mingallocated,1,MPIU_PETSCLOGDOUBLE,MPI_MIN,0,comm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Current space PetscMalloc()ed: total %5.4e max %5.4e min %5.4e\n",gallocated,maxgallocated,mingallocated);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Run with -memory_view to get maximum memory usage\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"OS cannot compute process memory\n");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"Run with -malloc to get statistics on PetscMalloc() calls\nOS cannot compute process memory\n");CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
