Bool lecode_tools_Init( int* argc, char** argv[] ) {
/* This init function tells StGermain of all the component types, etc this module contributes. Because it can be linked at compile
time or linked in by a toolbox at runtime, we need to make sure it isn't run twice (compiled in and loaded through a toolbox.*/
if( !ToolboxesManager_IsInitialised( stgToolboxesManager, "lecode_tools" ) ) {
int tmp;
char* directory;
lecode_tools_Base_Init(argc, argv);
Journal_Printf( Journal_Register( DebugStream_Type, (Name)"Context" ), "In: %s\n", __func__ ); /* DO NOT CHANGE OR REMOVE */
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), 0 );
Journal_Printf( /* DO NOT CHANGE OR REMOVE */
Journal_Register( InfoStream_Type, (Name)"Context" ),
"lecode_tools. Copyright (C) 2012 Monash University.\n" );
Stream_Flush( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), tmp );
/* Add the lecode_tools path to the global xml path dictionary */
directory = Memory_Alloc_Array( char, 200, "xmlDirectory" ) ;
sprintf(directory, "%s%s", LIB_DIR, "/StGermain" );
XML_IO_Handler_AddDirectory( "lecode_tools", directory );
Memory_Free(directory);
/* Add the plugin path to the global plugin list */
ModulesManager_AddDirectory( "lecode_tools", LIB_DIR );
return True;
}
/*
* Initialises the Linear Algebra package, then any init for this package
* such as streams etc.
*/
Bool StgFEM_Assembly_Init( int* argc, char** argv[] ) {
Stg_ComponentRegister* componentRegister = Stg_ComponentRegister_Get_ComponentRegister();
int tmp;
/* DO NOT CHANGE OR REMOVE */
Journal_Printf( Journal_Register( DebugStream_Type, (Name)"Context" ), "In: %s\n", __func__ );
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), 0 );
Stream_Flush( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), tmp );
/* initialise this level's streams */
StgFEM_Assembly_Debug = Stream_RegisterChild( StgFEM_Debug, "Assembly" );
Stg_ComponentRegister_Add( componentRegister, ThermalBuoyancyForceTerm_Type, (Name)"0", _ThermalBuoyancyForceTerm_DefaultNew );
Stg_ComponentRegister_Add( componentRegister, GradientStiffnessMatrixTerm_Type, (Name)"0", _GradientStiffnessMatrixTerm_DefaultNew );
Stg_ComponentRegister_Add( componentRegister, DivergenceMatrixTerm_Type, (Name)"0", _DivergenceMatrixTerm_DefaultNew );
Stg_ComponentRegister_Add( componentRegister, LaplacianStiffnessMatrixTerm_Type, (Name)"0", _LaplacianStiffnessMatrixTerm_DefaultNew );
Stg_ComponentRegister_Add( componentRegister, IsoviscousStressTensorTerm_Type, (Name)"0", _IsoviscousStressTensorTerm_DefaultNew );
Stg_ComponentRegister_Add( componentRegister, MassMatrixTerm_Type, (Name)"0", _MassMatrixTerm_DefaultNew );
RegisterParent( ThermalBuoyancyForceTerm_Type, ForceTerm_Type );
RegisterParent( GradientStiffnessMatrixTerm_Type, StiffnessMatrixTerm_Type );
RegisterParent( DivergenceMatrixTerm_Type, StiffnessMatrixTerm_Type );
RegisterParent( LaplacianStiffnessMatrixTerm_Type, StiffnessMatrixTerm_Type );
RegisterParent( IsoviscousStressTensorTerm_Type, StiffnessMatrixTerm_Type );
RegisterParent( MassMatrixTerm_Type, StiffnessMatrixTerm_Type );
return True;
}
Bool StGermainBase_Init( int* argc, char** argv[] ) {
char* directory;
int tmp;
/* Initialise enough bits and pieces to get IO going */
BaseFoundation_Init( argc, argv );
BaseIO_Init( argc, argv );
/* Write out the copyright message */
Journal_Printf( Journal_Register( DebugStream_Type, "Context" ), "In: %s\n", __func__ );
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), 0 );
Stream_Flush( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), tmp );
/* Initialise the remaining bits and pieces */
BaseContainer_Init( argc, argv );
BaseAutomation_Init( argc, argv );
BaseExtensibility_Init( argc, argv );
BaseContext_Init( argc, argv );
/* Add the StGermain path to the global xml path dictionary */
directory = Memory_Alloc_Array( char, 200, "xmlDirectory" ) ;
sprintf( directory, "%s%s", LIB_DIR, "/StGermain" );
XML_IO_Handler_AddDirectory( "StGermain", directory );
Memory_Free( directory );
/* Add the plugin path to the global plugin list */
ModulesManager_AddDirectory( "StGermain", LIB_DIR );
return True;
}
void Materials_Register_AssignParticleProperties(
void* materialRegister,
MaterialPointsSwarm* swarm,
Variable_Register* variableRegister )
{
Materials_Register* self = (Materials_Register*)materialRegister;
Material* material;
Particle_Index lParticle_I;
Particle_Index particleLocalCount = swarm->particleLocalCount;
Particle_Index particleGlobalCount = 0;
Stream* stream = Journal_Register( Info_Type, (Name)self->type );
Processor_Index formerStreamPrintingRank;
double setupStartTime = 0;
double setupTime = 0, tmin, tmax;
Bool firstStatusPrint = True;
formerStreamPrintingRank = Stream_GetPrintingRank( stream );
Stream_SetPrintingRank( stream, 0 );
Journal_Printf( stream, "In func %s(): for swarm \"%s\"\n", __func__, swarm->name );
Stream_Indent( stream );
setupStartTime = MPI_Wtime();
MPI_Reduce( &particleLocalCount, &particleGlobalCount, 1, MPI_UNSIGNED, MPI_SUM, 0, swarm->comm );
Journal_Printf( stream, "Assigning initial particle properties to the %u global particles\n",
particleGlobalCount );
Stream_Indent( stream );
for ( lParticle_I = 0 ; lParticle_I < particleLocalCount ; lParticle_I++ ) {
material = MaterialPointsSwarm_GetMaterialAt( swarm, lParticle_I );
if ( material ) {
/* Loop through material's dictionary assigning values to the variables of this particle */
Variable_Register_SetAllVariablesFromDictionary( variableRegister, lParticle_I, material->dictionary );
}
_Materials_Register_PrintParticleAssignUpdate( self, swarm, lParticle_I, stream, &firstStatusPrint );
}
Stream_UnIndent( stream );
/* Need this barrier so the time is accurate */
MPI_Barrier( swarm->comm );
setupTime = MPI_Wtime() - setupStartTime;
Stream_UnIndent( stream );
MPI_Reduce( &setupTime, &tmin, 1, MPI_DOUBLE, MPI_MIN, 0, swarm->comm );
MPI_Reduce( &setupTime, &tmax, 1, MPI_DOUBLE, MPI_MAX, 0, swarm->comm );
Journal_Printf( stream, "%s(): finished setup of material properties for swarm \"%s\"\n"
"\ttook %g [min] / %g [max] secs\n", __func__, swarm->name, tmin, tmax );
Stream_SetPrintingRank( stream, formerStreamPrintingRank );
}
/*
* Initialises the Linear Algebra package, then any init for this package
* such as streams etc.
*/
Bool StgFEM_SLE_Init( int* argc, char** argv[] ) {
int tmp;
/* DO NOT CHANGE OR REMOVE */
Journal_Printf( Journal_Register( DebugStream_Type, (Name)"Context" ), "In: %s\n", __func__ );
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), 0 );
Stream_Flush( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), tmp );
//StgFEM_SLE_LinearAlgebra_Init( argc, argv );
StgFEM_SLE_SystemSetup_Init( argc, argv );
StgFEM_SLE_ProvidedSystems_Init( argc, argv );
return True;
}
Bool Discretisation_Init( int* argc, char** argv[] ) {
int tmp;
Journal_Printf( Journal_Register( DebugStream_Type, "Context" ), "In: %s\n", __func__ ); /* DO NOT CHANGE OR REMOVE */
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), 0 );
Journal_Printf( /* DO NOT CHANGE OR REMOVE */
Journal_Register( InfoStream_Type, "Context" ),
"StGermain Discretisation Library revision %s. Copyright (C) 2003-2005 VPAC.\n", VERSION );
Stream_Flush( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), tmp );
DiscretisationGeometry_Init( argc, argv );
DiscretisationShape_Init( argc, argv );
DiscretisationMesh_Init( argc, argv );
DiscretisationUtils_Init( argc, argv );
DiscretisationSwarm_Init( argc, argv );
return True;
}
Bool Geothermal_Init( int* argc, char** argv[] ) {
/*
* This init function tells StGermain of all the component types, etc this module contributes.
* Because it can be linked at compile time or linked in by a toolbox at runtime, we need to make
* sure it isn't run twice (compiled in and loaded through a toolbox.
*/
if( !ToolboxesManager_IsInitialised( stgToolboxesManager, "Geothermal" ) ) {
int tmp;
char* directory;
Geothermal_Base_Init(argc, argv);
/* DO NOT CHANGE OR REMOVE */
Journal_Printf( Journal_Register( DebugStream_Type, (Name)"Context" ), "In: %s\n", __func__ );
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), 0 );
/* DO NOT CHANGE OR REMOVE */
Journal_Printf(
Journal_Register( InfoStream_Type, (Name)"Context" ),
"Geothermal (Bleeding Edge Geodynamics framework) Revision %s. Copyright (C) 2005 Monash University.\n",
VERSION );
/* Add repo indentity info in the repo dictionary. */
Dictionary_Add( versionDict, "Geothermal", Dictionary_Entry_Value_FromString( VERSION ) );
Dictionary_Add( branchDict, "Geothermal", Dictionary_Entry_Value_FromString( BRANCH ) );
Dictionary_Add( pathDict, "Geothermal", Dictionary_Entry_Value_FromString( PATH ) );
Stream_Flush( Journal_Register( InfoStream_Type, (Name)"Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, (Name)"Context" ), tmp );
/* Add the Geothermal path to the global xml path dictionary */
directory = Memory_Alloc_Array( char, 200, "xmlDirectory" ) ;
sprintf(directory, "%s%s", LIB_DIR, "/StGermain" );
XML_IO_Handler_AddDirectory( "Geothermal", directory );
Memory_Free(directory);
/* Add the plugin path to the global plugin list */
ModulesManager_AddDirectory( "Geothermal", LIB_DIR );
return True;
}
void _Stokes_SLE_UzawaSolver_Solve( void* solver, void* stokesSLE ) {
Stokes_SLE_UzawaSolver* self = (Stokes_SLE_UzawaSolver*)solver;
Stokes_SLE* sle = (Stokes_SLE*)stokesSLE;
/* Create shortcuts to stuff needed on sle */
Mat K_Mat = sle->kStiffMat->matrix;
Mat G_Mat = sle->gStiffMat->matrix;
Mat D_Mat = NULL;
Mat M_Mat = NULL;
Vec uVec = sle->uSolnVec->vector;
Vec qVec = sle->pSolnVec->vector;
Vec fVec = sle->fForceVec->vector;
Vec hVec = sle->hForceVec->vector;
/* Create shortcuts to solver related stuff */
Vec qTempVec = self->pTempVec;
Vec rVec = self->rVec;
Vec sVec = self->sVec;
Vec fTempVec = self->fTempVec;
Vec vStarVec = self->vStarVec;
KSP velSolver = self->velSolver; /* Inner velocity solver */
KSP pcSolver = self->pcSolver; /* Preconditioner */
Iteration_Index maxIterations = self->maxUzawaIterations;
Iteration_Index minIterations = self->minUzawaIterations;
Iteration_Index iteration_I = 0;
Iteration_Index outputInterval = 1;
double zdotr_current = 0.0;
double zdotr_previous = 1.0;
double sdotGTrans_v;
double alpha, beta;
double absResidual;
double relResidual;
double* chosenResidual; /* We can opt to use either the absolute or relative residual in termination condition */
double uzawaRhsScale;
double divU;
double weightedResidual;
double weightedVelocityScale;
double momentumEquationResidual;
Iteration_Index innerLoopIterations;
Stream* errorStream = Journal_Register( Error_Type, (Name)Stokes_SLE_UzawaSolver_Type );
PetscInt fVecSize, qTempVecSize, uVecSize, qVecSize;
PetscScalar fVecNorm, qTempVecNorm, uVecNorm, rVecNorm, fTempVecNorm, uVecNormInf, qVecNorm, qVecNormInf;
double qGlobalProblemScale;
double qReciprocalGlobalProblemScale;
int init_info_stream_rank;
PetscScalar p_sum;
/* Bool nullsp_present; */
Bool uzawa_summary;
double time,t0,rnorm0;
PetscTruth flg;
double ksptime;
VecGetSize( qTempVec, &qTempVecSize );
qGlobalProblemScale = sqrt( (double) qTempVecSize );
qReciprocalGlobalProblemScale = 1.0 / qGlobalProblemScale;
init_info_stream_rank = Stream_GetPrintingRank( self->info );
Stream_SetPrintingRank( self->info, 0 );
/* DEFINITIONS:
See accompanying documentation
u - the displacement / velocity solution (to which constraints are applied)
q - the pressure-like variable which constrains the divergence displacement / velocity (= pressure for incompressible)
F - standard FE force vector
Fhat - Uzawa RHS = K^{-1} G F - h
K - standard FE stiffness matrix
Khat - Uzawa transformed stiffness matrix = G^T K^{-1} G
G matrix - discrete gradient operator
D matrix - discrete divergence operator = G^T for this particular algorithm
C matrix - Mass matrix (M) for compressibility
LM & DAM
*/
/* CHOICE OF RESIDUAL:
we may opt to converge on the absolute value (self->useAbsoluteTolerance == True ... default)
or the relative value of the residual (self->useAbsoluteTolerance == False)
(another possibility would be always to improve the residual by a given tolerance)
The Moresi & Solomatov (Phys Fluids, 1995) approach is to use the relative tolerance
*/
VecNorm( fVec, NORM_2, &fVecNorm );
VecGetSize( fVec, &fVecSize );
if ( fVecNorm / sqrt( (double)fVecSize ) <= 1e-99 ) {
Journal_Printf( errorStream,
"Error in func %s: The momentum force vector \"%s\" is zero. "
"The force vector should be non-zero either because of your chosen boundary "
"conditions, or because of the element force vector assembly. You have %d "
"element force vectors attached.\n",
__func__, sle->fForceVec->name, sle->fForceVec->assembleForceVector->hooks->count );
if ( sle->fForceVec->assembleForceVector->hooks->count > 0 ) {
Journal_Printf( errorStream, "You used the following force vector assembly terms:\n" );
EntryPoint_PrintConcise( sle->fForceVec->assembleForceVector, errorStream );
/* TODO : need to print the elementForceVector assembly, not the global guy!! */
}
Journal_Printf( errorStream,
"Please check values for building the force vector.\n" );
Journal_Firewall( 0, errorStream, "Exiting.\n" );
}
Journal_DPrintf( self->debug, "In %s:\n", __func__ );
Journal_RPrintfL( self->debug, 2, "Conjugate Gradient Uzawa solver with:\n");
Stream_IndentBranch( StgFEM_Debug );
Journal_RPrintfL( self->debug, 2, "Compressibility %s\n", (sle->cStiffMat)? "on" : "off");
Journal_RPrintfL( self->debug, 2, "Preconditioning %s\n", (pcSolver)? "on" : "off" );
if ( sle->cStiffMat ) {
Journal_DPrintfL( self->debug, 2, "(compressibility active)\n" );
M_Mat = sle->cStiffMat->matrix;
}
else {
Journal_DPrintfL( self->debug, 2, "(compressibility inactive)\n" );
}
if ( sle->dStiffMat ) {
Journal_DPrintfL( self->debug, 2, "(asymmetric geometry: handling D Matrix [incorrectly - will be ignored])\n" );
D_Mat = sle->dStiffMat->matrix;
}
else {
Journal_DPrintfL( self->debug, 2, "(No D -> symmetric geometry: D = Gt)\n" );
}
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 3 ) ) {
Journal_DPrintf( self->debug, "Matrices and Vectors to solve are:\n" );
Journal_DPrintf( self->debug, "K Matrix:\n" );
/* No nice way of viewing Matrices, so commented out as incompatible with
* new 3D decomp at present --Kathleen Humble 30-04-07
* Matrix_View( sle->kStiffMat->matrix, self->debug ); */
Journal_DPrintf( self->debug, "G Matrix:\n" );
if ( D_Mat ) {
Journal_DPrintf( self->debug, "D Matrix:\n" );
}
if ( M_Mat ) {
Journal_DPrintf( self->debug, "M Matrix:\n" );
}
Journal_DPrintf( self->debug, "Z (preconditioner) Matrix:\n" );
Journal_DPrintf( self->debug, "f Vector:\n" );
_SLE_VectorView( fVec, self->debug );
Journal_DPrintf( self->debug, "h Vector:\n" );
_SLE_VectorView( hVec, self->debug );
}
#endif
/* STEP 1: Estimate the magnitude of the RHS for the transformed problem
we compute (usually to lower accuracy than elsewhere) the RHS (Fhat - h)
and store the result in qTempVec.
LM & DAM
*/
Journal_DPrintfL( self->debug, 2, "Building Fhat - h.\n" );
PetscOptionsHasName(PETSC_NULL,"-uzawa_printksptimes",&flg);
KSPSetTolerances( velSolver, self->tolerance, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT );
if (flg) {
ksptime = MPI_Wtime();
}
KSPSolve( velSolver, fVec, vStarVec );
if (flg) {
ksptime = MPI_Wtime() - ksptime;
PetscPrintf( PETSC_COMM_WORLD, "KSP on velSolver took %lf seconds in Building Fhat step\n", ksptime);
}
KSPGetIterationNumber( velSolver, &innerLoopIterations );
Journal_DPrintfL( self->debug, 2, "Fhat inner solution: Number of iterations: %d\n", innerLoopIterations );
if ( D_Mat ) {
MatMult( D_Mat, vStarVec, qTempVec );
}
else {
MatMultTranspose( G_Mat, vStarVec, qTempVec );
}
VecAXPY( qTempVec, -1.0, hVec );
/* WARNING:
If D != G^T then the resulting \hat{K} is not likely to be symmetric, positive definite as
required by this implementation of the Uzawa iteration. This next piece of code
is VERY unlikely to work properly so it's in the sin bin for the time being - LM.
if ( D_Mat ) {
MatrixMultiply( D_Mat, vStarVec, qTempVec );
}
else {
MatrixTransposeMultiply( G_Mat, vStarVec, qTempVec );
}
LM & DAM
*/
/* STEP 2: The problem scaling - optionally normalize the uzawa residual by the magnitude of the RHS (use a relative tolerance)
For the inner velocity solver, Citcom uses a relative tolerance equal to that used for the Uzawa iteration as a whole
LM & DAM
*/
if (self->useAbsoluteTolerance) {
chosenResidual = &absResidual;
Journal_PrintfL( self->info, 2, "Absolute residual < %g for Uzawa stopping condition\n", self->tolerance);
/* We should calculate the effective relative tolerance and insert that here !! */
KSPSetTolerances( velSolver, 0.1 * self->tolerance, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT );
}
else { /* The CITCOM compatible choice */
chosenResidual = &relResidual;
Journal_PrintfL( self->info, 2, "Relative residual < %g for Uzawa stopping condition\n", self->tolerance);
KSPSetTolerances( velSolver, 0.1 * self->tolerance, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT );
}
Journal_DPrintfL( self->debug, 2, "Determining scaling factor for residual:\n" );
VecNorm( qTempVec, NORM_2, &qTempVecNorm );
uzawaRhsScale = ((double)qTempVecNorm) * qReciprocalGlobalProblemScale;
Journal_DPrintfL( self->debug, 2, "uzawaRhsScale = %f\n", uzawaRhsScale );
Journal_Firewall( isGoodNumber( uzawaRhsScale ), errorStream,
"Error in func '%s' for %s '%s' - uzawaRhsScale has illegal value '%g'.\n", __func__, self->type, self->name, uzawaRhsScale );
/* STEP 3: Calculate initial residual for transformed equation (\hat{F} - h - \hat{K} q_0)
Compute the solution to K u_0 = F - G q_0 (u_0 unknown)
Then G^T u* = \hat{F} - \hat{K} q_0
u_0 is also the initial velocity solution to which the constraint is applied by the subsequent iteration
LM & DAM
*/
Journal_DPrintfL( self->debug, 2, "Solving for transformed Uzawa RHS.\n" );
VecCopy( fVec, fTempVec );
VecScale( fTempVec, -1.0 );
MatMultAdd( G_Mat, qVec, fTempVec, fTempVec );
VecScale( fTempVec, -1.0 );
KSPSolve( velSolver, fTempVec, uVec );
/* Handling for NON-SYMMETRIC: relegated to sin bin (see comment above) LM & DAM */
if ( D_Mat ) {
MatMult( D_Mat, uVec, rVec );
}
else {
MatMultTranspose( G_Mat, uVec, rVec );
}
VecNorm( rVec, NORM_2, &rnorm0 );
VecNorm( uVec, NORM_2, &uVecNorm );
divU = rnorm0 / uVecNorm;
Journal_PrintfL( self->info, 2, "Initial l2Norm( Div u ) / l2Norm( u ) = %f \n", divU);
Journal_Firewall( isGoodNumber( divU ), errorStream,
"Error in func '%s' for %s '%s' - l2Norm( Div u ) has illegal value '%g'.\n",
__func__, self->type, self->name, divU );
Journal_DPrintfL( self->debug, 2, "Adding compressibility and prescribed divergence terms.\n" );
if ( M_Mat ) {
MatMultAdd( M_Mat, qVec, rVec, rVec );
}
VecAXPY( rVec, -1.0, hVec );
/* Check for existence of constant null space */
#if 0
nullsp_present = _check_if_constant_nullsp_present( self, K_Mat,G_Mat,M_Mat, fTempVec,vStarVec,qTempVec,sVec, velSolver );
#endif
/* STEP 4: Preconditioned conjugate gradient iteration loop */
Journal_DPrintfL( self->debug, 1, "Beginning main Uzawa conjugate gradient loop:\n" );
iteration_I = 0;
/* outer_it, residual, time */
uzawa_summary = self->monitor;
time = 0.0;
t0 = MPI_Wtime();
// Journal_PrintfL( self->info, 1, " |r0| = %.8e \n", rnorm0 );
do{
/* reset initial time and end time for inner its back to 0 - probs don't need to do this but just in case */
self->outeritsinitialtime = 0;
self->outeritsendtime = 0;
//BEGINNING OF OUTER ITERATIONS!!!!!
/*get wall time for start of outer loop*/
self->outeritsinitialtime = MPI_Wtime();
Journal_DPrintfL( self->debug, 2, "Beginning solve '%u'.\n", iteration_I );
Stream_IndentBranch( StgFEM_Debug );
/* STEP 4.1: Preconditioner
Solve:
Q_\hat{K} z_1 = r_1
Q_\hat{K} is an approximation to \hat{K} which is simple / trivial / quick to invert
LM & DAM
*/
if ( pcSolver ) {
PetscOptionsHasName(PETSC_NULL,"-uzawa_printksptimes",&flg);
if (flg) {
ksptime = MPI_Wtime();
}
KSPSolve( pcSolver, rVec, qTempVec );
if (flg) {
ksptime = MPI_Wtime() - ksptime;
PetscPrintf( PETSC_COMM_WORLD, "KSP on pcSolver took %lf seconds\n", ksptime);
}
}
else {
VecCopy( rVec, qTempVec );
}
/* Remove the constant null space, but only if NOT compressible */
#if 0
if( nullsp_present == True ) {
_remove_constant_nullsp( qTempVec );
}
#endif
/* STEP 4.2: Calculate s_I, the pressure search direction
z_{I-1} . r_{I-1}
\beta = (z_{I-1} . r_{I-1}) / (z_{I-2} . r_{I-2})
\beta = 0 for the first iteration
s_I = z_(I-1) + \beta * s_(I-1)
LM & DAM
*/
VecDot( qTempVec, rVec, &zdotr_current );
VecNorm( qTempVec, NORM_2, &qTempVecNorm );
VecNorm( rVec, NORM_2, &rVecNorm );
Journal_DPrintfL( self->debug, 2, "l2Norm (qTempVec) %g; (rVec) %g \n",
qTempVecNorm * qReciprocalGlobalProblemScale,
rVecNorm * qReciprocalGlobalProblemScale );
if ( iteration_I == 0 ) {
VecCopy( qTempVec, sVec );
}
else {
beta = zdotr_current/zdotr_previous;
VecAYPX( sVec, beta, qTempVec );
}
/* STEP 4.3: Velocity search direction corresponding to s_I is found by solving
K u* = G s_I
LM & DAM
*/
MatMult( G_Mat, sVec, fTempVec );
Journal_DPrintfL( self->debug, 2, "Uzawa inner iteration step\n");
//START OF INNER ITERATIONS!!!!
PetscOptionsHasName(PETSC_NULL,"-uzawa_printksptimes",&flg);
/*get initial wall time for inner loop*/
self->inneritsinitialtime = MPI_Wtime();
if (flg) {
ksptime = MPI_Wtime();
}
KSPSolve( velSolver, fTempVec, vStarVec );
if (flg) {
ksptime = MPI_Wtime() - ksptime;
PetscPrintf( PETSC_COMM_WORLD, "KSP on velSolver took %lf seconds in Uzawa inner iteration step\n", ksptime);
}
/*get end wall time for inner loop*/
self->inneritsendtime = MPI_Wtime();
/* add time to total time inner its: */
self->totalinneritstime = self->totalinneritstime + (-self->inneritsinitialtime + self->inneritsendtime);
/* reset initial time and end time for inner its back to 0 - probs don't need to do this but just in case */
self->inneritsinitialtime = 0;
self->inneritsendtime = 0;
KSPGetIterationNumber( velSolver, &innerLoopIterations );
/* add the inner loop iterations to the total inner iterations */
self->totalnuminnerits = self->totalnuminnerits + innerLoopIterations;
Journal_DPrintfL( self->debug, 2, "Completed Uzawa inner iteration in '%u' iterations \n", innerLoopIterations );
/* STEP 4.4: Calculate the step size ( \alpha = z_{I-1} . r_{I-1} / (s_I . \hat{K} s_I) )
\hat{K} s_I = G^T u* - M s_I (u* from step 4.3)
LM & DAM
*/
if ( D_Mat ) {
MatMult( D_Mat, vStarVec, qTempVec );
}
else {
MatMultTranspose( G_Mat, vStarVec, qTempVec );
}
/* Handling for NON-SYMMETRIC: relegated to sin bin (see comment above)
if ( D_Mat ) {
MatrixMultiply( D_Mat, vStarVec, qTempVec );
}
else {
MatrixTransposeMultiply( G_Mat, vStarVec, qTempVec );
}
LM & DAM
*/
if ( M_Mat ) {
Journal_DPrintfL( self->debug, 2, "Correcting for Compressibility\n" );
VecScale( qTempVec, -1.0 );
MatMultAdd( M_Mat, sVec, qTempVec, qTempVec );
VecScale( qTempVec, -1.0 );
}
VecDot( sVec, qTempVec, &sdotGTrans_v );
alpha = zdotr_current/sdotGTrans_v;
/* STEP 4.5: Update pressure, velocity and value of residual
by \alpha times corresponding search direction
LM & DAM
*/
Journal_DPrintfL( self->debug, 2, "zdotr_current = %g \n", zdotr_current);
Journal_DPrintfL( self->debug, 2, "sdotGTrans_v = %g \n", sdotGTrans_v);
Journal_DPrintfL( self->debug, 2, "alpha = %g \n", alpha);
Journal_Firewall(
isGoodNumber( zdotr_current ) && isGoodNumber( sdotGTrans_v ) && isGoodNumber( alpha ),
errorStream,
"Error in func '%s' for %s '%s' - zdotr_current, sdotGTrans_v or alpha has an illegal value: '%g','%g' or '%g'\n",
__func__, self->type, self->name, zdotr_current, sdotGTrans_v, alpha );
VecAXPY( qVec, alpha, sVec );
VecAXPY( uVec, -alpha, vStarVec );
VecAXPY( rVec, -alpha, qTempVec );
/* STEP 4.6: store the value of z_{I-1} . r_{I-1} for the next iteration
LM & DAM
*/
zdotr_previous = zdotr_current;
VecNorm( rVec, NORM_2, &rVecNorm );
absResidual = rVecNorm * qReciprocalGlobalProblemScale;
relResidual = absResidual / uzawaRhsScale;
Stream_UnIndentBranch( StgFEM_Debug );
if( iteration_I % outputInterval == 0 ) {
Journal_PrintfL( self->info, 2, "\tLoop = %u, absResidual = %.8e, relResidual = %.8e\n",
iteration_I, absResidual, relResidual );
}
Journal_Firewall( isGoodNumber( absResidual ), errorStream,
"Error in func '%s' for %s '%s' - absResidual has an illegal value: '%g'\n",
__func__, self->type, self->name, absResidual );
Journal_Firewall( iteration_I < maxIterations,
errorStream, "In func %s: Reached maximum number of iterations %u without converging; absResidual = %.5g, relResidual = %.5g \n",
__func__, iteration_I, absResidual, relResidual );
/* TODO: test for small change in 10 iterations and if so restart? */
time = MPI_Wtime()-t0;
if (uzawa_summary) {
Journal_PrintfL( self->info, 1, " %1.4d uzawa residual norm %12.13e, cpu time %5.5e\n", iteration_I+1,*chosenResidual,time );
}
iteration_I++;
//END OF OUTER ITERATION LOOP!!!
/*get wall time for end of outer loop*/
self->outeritsendtime = MPI_Wtime();
/* add time to total time inner its: */
self->totalouteritstime = self->totalouteritstime + (-self->outeritsinitialtime + self->outeritsendtime);
/* reset initial time and end time for inner its back to 0 - probs don't need to do this but just in case */
self->outeritsinitialtime = 0;
self->outeritsendtime = 0;
/* add the outer loop iterations to the total outer iterations */
self->totalnumouterits++;
} while ( (*chosenResidual > self->tolerance) || (iteration_I<minIterations) );
// } while ( *chosenResidual > self->tolerance );
Journal_DPrintfL( self->debug, 1, "Pressure solution converged. Exiting uzawa \n ");
/* STEP 5: Check all the relevant residuals and report back */
if (Stream_IsEnable( self->info ) ) {
/* This information should be in an info stream */
Journal_PrintfL( self->info, 1, "Summary:\n");
Journal_PrintfL( self->info, 1, " Uzawa its. = %04d , Uzawa residual = %12.13e\n", iteration_I, relResidual );
MatMultTranspose( G_Mat, uVec, rVec );
VecNorm( rVec, NORM_2, &rVecNorm );
VecNorm( uVec, NORM_2, &uVecNorm );
divU = rVecNorm / uVecNorm;
Journal_PrintfL( self->info, 1, " |G^T u|/|u| = %.8e\n", divU);
/* Residual for the momentum equation
Compute r = || F - Ku - Gp || / || F ||
*/
MatMult( G_Mat, qVec, vStarVec );
MatMultAdd( K_Mat, uVec, vStarVec, fTempVec );
VecAYPX( fTempVec, -1.0, fVec );
VecNorm( fTempVec, NORM_2, &fTempVecNorm );
VecNorm( fVec, NORM_2, &fVecNorm );
momentumEquationResidual = fTempVecNorm / fVecNorm;
Journal_PrintfL( self->info, 1, " |f - K u - G p|/|f| = %.8e\n", momentumEquationResidual );
Journal_Firewall( isGoodNumber( momentumEquationResidual ), errorStream,
"Bad residual for the momentum equation (|| F - Ku - Gp || / || F || = %g):\n"
"\tCheck to see if forcing term is zero or nan - \n\t|| F - Ku - Gp || = %g \n\t|| F || = %g.\n",
momentumEquationResidual,
fTempVecNorm, fVecNorm );
/* "Preconditioned" residual for the momentum equation
r_{w} = || Q_{K}(r) || / || Q_{K}(F)
fTempVec contains the residual but is overwritten once used
vStarVec is used to hold the diagonal preconditioner Q_{K}
*/
MatGetDiagonal( K_Mat, vStarVec );
VecReciprocal( vStarVec );
VecPointwiseMult( vStarVec, fTempVec, fTempVec );
VecNorm( fTempVec, NORM_2, &weightedResidual );
VecPointwiseMult( vStarVec, fVec, fTempVec );
VecNorm( fTempVec, NORM_2, &weightedVelocityScale );
Journal_PrintfL( self->info, 1, " |f - K u - G p|_w/|f|_w = %.8e\n", weightedResidual / weightedVelocityScale );
/* Report back on the solution - velocity and pressure
Note - correction for dof in Vrms ??
*/
VecNorm( uVec, NORM_INFINITY, &uVecNormInf );
VecNorm( uVec, NORM_2, &uVecNorm );
VecGetSize( uVec, &uVecSize );
VecNorm( qVec, NORM_INFINITY, &qVecNormInf );
VecNorm( qVec, NORM_2, &qVecNorm );
VecGetSize( qVec, &qVecSize );
Journal_PrintfL( self->info, 1, " |u|_{\\infty} = %.8e , u_rms = %.8e\n",
uVecNormInf, uVecNorm / sqrt( (double)uVecSize ) );
Journal_PrintfL( self->info, 1, " |p|_{\\infty} = %.8e , p_rms = %.8e\n",
qVecNormInf, qVecNorm / sqrt( (double)qVecSize ) );
{ PetscInt lmin,lmax;
PetscReal min,max;
VecMax( uVec, &lmax, &max );
VecMin( uVec, &lmin, &min );
Journal_PrintfL( self->info, 1, " min/max(u) = %.8e [%d] / %.8e [%d]\n",min,lmin,max,lmax);
VecMax( qVec, &lmax, &max );
VecMin( qVec, &lmin, &min );
Journal_PrintfL( self->info, 1, " min/max(p) = %.8e [%d] / %.8e [%d]\n",min,lmin,max,lmax);
}
VecSum( qVec, &p_sum );
Journal_PrintfL( self->info, 1, " \\sum_i p_i = %.8e \n", p_sum );
} /* journal stream enabled */
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 3 ) ) {
Journal_DPrintf( self->debug, "Velocity solution:\n" );
_SLE_VectorView( uVec, self->debug );
Journal_DPrintf( self->debug, "Pressure solution:\n" );
_SLE_VectorView( qVec, self->debug );
}
#endif
Stream_UnIndentBranch( StgFEM_Debug );
Stream_SetPrintingRank( self->info, init_info_stream_rank );
/* Now gather up data for printing out to FrequentOutput file: */
/*!!! if non-linear need to divide by number of nonlinear iterations and we do this in SystemLinearEquations */
if((sle->isNonLinear != True)){
self->avgnuminnerits = self->totalnuminnerits/self->totalnumouterits;
self->avgnumouterits = self->totalnumouterits;
self->avgtimeouterits = (self->totalouteritstime - self->totalinneritstime)/self->totalnumouterits;
self->avgtimeinnerits = self->totalinneritstime/self->totalnuminnerits;
}
}
/* Main */
int main( int argc, char* argv[] ) {
MPI_Comm CommWorld;
int rank;
int numProcessors;
int procToWatch;
Dictionary* dictionary;
Dictionary* componentDict;
XML_IO_Handler* ioHandler;
char* filename;
Snac_Context* snacContext;
int tmp;
/* Initialise MPI, get world info */
MPI_Init( &argc, &argv );
MPI_Comm_dup( MPI_COMM_WORLD, &CommWorld );
MPI_Comm_size( CommWorld, &numProcessors );
MPI_Comm_rank( CommWorld, &rank );
if( argc >= 3 ) {
procToWatch = atoi( argv[2] );
}
else {
procToWatch = 0;
}
if( rank == procToWatch ) printf( "Watching rank: %i\n", rank );
if (!Snac_Init( &argc, &argv )) {
fprintf(stderr, "Error initialising StGermain, exiting.\n" );
exit(EXIT_FAILURE);
}
/* Snac's init message */
tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), 0 );
Journal_Printf( /* DO NOT CHANGE OR REMOVE */
Journal_Register( InfoStream_Type, "Context" ),
"Snac. Copyright (C) 2003-2005 Caltech, VPAC & University of Texas.\n" );
Stream_Flush( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), tmp );
MPI_Barrier( CommWorld ); /* Ensures copyright info always come first in output */
/* Create the dictionary, and some fixed values */
dictionary = Dictionary_New();
Dictionary_Add( dictionary, "rank", Dictionary_Entry_Value_FromUnsignedInt( rank ) );
Dictionary_Add( dictionary, "numProcessors", Dictionary_Entry_Value_FromUnsignedInt( numProcessors ) );
/* Read input */
ioHandler = XML_IO_Handler_New();
if( argc >= 2 ) {
filename = strdup( argv[1] );
}
else {
filename = strdup( "input.xml" );
}
if ( False == IO_Handler_ReadAllFromFile( ioHandler, filename, dictionary ) )
{
fprintf( stderr, "Error: Snac couldn't find specified input file %s. Exiting.\n", filename );
exit( EXIT_FAILURE );
}
Journal_ReadFromDictionary( dictionary );
snacContext = Snac_Context_New( 0.0f, 0.0f, sizeof(Snac_Node), sizeof(Snac_Element), CommWorld, dictionary );
if( rank == procToWatch ) Dictionary_PrintConcise( dictionary, snacContext->verbose );
/* Construction phase -----------------------------------------------------------------------------------------------*/
Stg_Component_Construct( snacContext, 0 /* dummy */, &snacContext, True );
/* Building phase ---------------------------------------------------------------------------------------------------*/
Stg_Component_Build( snacContext, 0 /* dummy */, False );
/* Initialisaton phase ----------------------------------------------------------------------------------------------*/
Stg_Component_Initialise( snacContext, 0 /* dummy */, False );
if( rank == procToWatch ) Context_PrintConcise( snacContext, snacContext->verbose );
/* Step the context solver */
Stg_Component_Execute( snacContext, 0 /* dummy */, False );
/* Stg_Class_Delete stuff */
Stg_Component_Destroy( snacContext, 0 /* dummy */, False );
Stg_Class_Delete( snacContext );
free( filename );
Stg_Class_Delete( ioHandler );
Stg_Class_Delete( dictionary );
/* Close off frameworks */
Snac_Finalise();
MPI_Finalize();
return 0; /* success */
}
int
BMI_Initialize (const char *config_file, BMI_Model ** handle)
{
BMI_Model *self;
MPI_Comm CommWorld;
int rank;
int numProcessors;
int procToWatch;
char* filename;
if (!handle)
return BMI_FAILURE;
self = malloc( sizeof(BMI_Model) );
/* Initialise MPI, get world info */
MPI_Init( NULL, NULL ); /* MPI_Init( &argc, &argv ); */
MPI_Comm_dup( MPI_COMM_WORLD, &CommWorld );
MPI_Comm_size( CommWorld, &numProcessors );
MPI_Comm_rank( CommWorld, &rank );
/* Hardwire the process to watch to 0.
Note that it doesn't have to be 0 when multiple processrs are used. */
procToWatch = 0;
#if 0
if( argc >= 3 ) {
procToWatch = atoi( argv[2] );
}
else {
procToWatch = 0;
}
#endif
if( rank == procToWatch ) printf( "Watching rank: %i\n", rank );
/* if (!Snac_Init( &argc, &argv )) { */
if (!Snac_Init( NULL, NULL )) {
fprintf(stderr, "Error initialising StGermain, exiting.\n" );
exit(EXIT_FAILURE);
}
/* Snac's init message */
{
int tmp = Stream_GetPrintingRank( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), 0 );
Journal_Printf( /* DO NOT CHANGE OR REMOVE */
Journal_Register( InfoStream_Type, "Context" ),
"Snac. Copyright (C) 2003-2005 Caltech, VPAC & University of Texas.\n" );
Stream_Flush( Journal_Register( InfoStream_Type, "Context" ) );
Stream_SetPrintingRank( Journal_Register( InfoStream_Type, "Context" ), tmp );
}
/* Ensures copyright info always come first in output */
MPI_Barrier( CommWorld );
/* Create the dictionary, and some fixed values */
/* Hardwirred to the one-processor scenario for now. */
self->dictionary = Dictionary_New();
Dictionary_Add( self->dictionary, "rank", Dictionary_Entry_Value_FromUnsignedInt( 0 ) );
Dictionary_Add( self->dictionary, "numProcessors", Dictionary_Entry_Value_FromUnsignedInt( 1 ) );
/* Read input */
self->ioHandler = XML_IO_Handler_New();
filename = strdup( config_file );
if ( False == IO_Handler_ReadAllFromFile( self->ioHandler, filename, self->dictionary ) )
{
fprintf( stderr, "Error: Snac couldn't find specified input file %s. Exiting.\n", filename );
exit( EXIT_FAILURE );
}
Journal_ReadFromDictionary( self->dictionary );
free( filename );
/* This is the handle to the SNAC's model data. */
self->snacContext = Snac_Context_New( 0.0f, 0.0f, sizeof(Snac_Node), sizeof(Snac_Element), CommWorld, self->dictionary );
if( rank == procToWatch ) Dictionary_PrintConcise( self->dictionary, self->snacContext->verbose );
/* Construction phase -----------------------------------------------------------------------------------------------*/
Stg_Component_Construct( self->snacContext, 0 /* dummy */, &(self->snacContext), True );
/* Building phase ---------------------------------------------------------------------------------------------------*/
Stg_Component_Build( self->snacContext, 0 /* dummy */, False );
/* Initialisaton phase ----------------------------------------------------------------------------------------------*/
Stg_Component_Initialise( self->snacContext, 0 /* dummy */, False );
if( rank == procToWatch ) Context_PrintConcise( self->snacContext, self->snacContext->verbose );
/* pass the pointer to Snac_Context to handle */
*handle = self;
return BMI_SUCCESS;
}
