Epetra_CrsMatrix *
NOX::Epetra::DebugTools::compute_matrix_using_operator( const Epetra_Operator * op)
{
const Epetra_Map & rowMap = op->OperatorDomainMap();
Epetra_CrsMatrix * p_mat = new Epetra_CrsMatrix(Copy, rowMap, 0);
Epetra_Vector * tempVec = new Epetra_Vector(rowMap);
Epetra_Vector * tempRes = new Epetra_Vector(rowMap);
// Show progress on what could be a long operation
if( 0 == op->Comm().MyPID() )
{
std::cout << "**************** CREATING MATRIX FROM OPERATOR ************************ "
<< std::endl;
std::cout << NOX::Utils::fill(72) << std::endl;
}
int totalPerturbations = tempVec->GlobalLength();
int outFreq = totalPerturbations / 71;
if( 1 > outFreq )
outFreq = 1;
for( int col = 0; col < tempVec->GlobalLength(); ++col )
{
tempVec->PutScalar(0.0);
if( rowMap.MyGID(col) )
(*tempVec)[col] = 1.0;
op->Apply(*tempVec, *tempRes);
// Fill in only the nonzero entries from the apply
for( int row = 0; row < p_mat->NumMyRows(); ++row)
{
if( fabs( (*tempRes)[row] ) > 1.e-12 )
{
int ierr = p_mat->InsertGlobalValues( rowMap.GID(row), 1, &(*tempRes)[row], &col );
if( ierr < 0 )
{
std::string msg = //"ERROR (" + ierr + ") : "
"NOX::Epetra::DebugTools::compute_matrix_using_operator crsMatrix.ExtractGlobalRowView(...) failed for row : ";// + row;
throw msg;
}
}
}
if( (0 == op->Comm().MyPID()) && (0 == (col % outFreq)) )
std::cout << "-" << std::flush;
}
if( 0 == op->Comm().MyPID() )
std::cout << "*" << std::endl;
p_mat->FillComplete();
delete tempRes;
delete tempVec;
return p_mat;
}
void EpetraLinearOp::computeAbsRowSum(Epetra_Vector & x) const
{
TEUCHOS_ASSERT(!is_null(rowMatrix_));
RCP<Epetra_CrsMatrix> crsMatrix
= Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(rowMatrix_);
TEUCHOS_TEST_FOR_EXCEPTION(is_null(crsMatrix),
Exceptions::OpNotSupported,
"EpetraLinearOp::computeAbsRowSum(...): wrapped matrix must be of type "
"Epetra_CrsMatrix for this method. Other operator types are not supported."
);
//
// Put inverse of the sum of absolute values of the ith row of A in x[i].
// (this is a modified copy of Epetra_CrsMatrix::InvRowSums)
//
if (crsMatrix->Filled()) {
TEUCHOS_TEST_FOR_EXCEPTION(is_null(crsMatrix),
std::invalid_argument,
"EpetraLinearOp::computeAbsRowSum(...): Epetra_CrsMatrix must be filled"
);
}
int i, j;
x.PutScalar(0.0); // Make sure we sum into a vector of zeros.
double * xp = (double*)x.Values();
if (crsMatrix->Graph().RangeMap().SameAs(x.Map()) && crsMatrix->Exporter() != 0) {
Epetra_Vector x_tmp(crsMatrix->RowMap());
x_tmp.PutScalar(0.0);
double * x_tmp_p = (double*)x_tmp.Values();
for (i=0; i < crsMatrix->NumMyRows(); i++) {
int NumEntries = 0;
double * RowValues = 0;
crsMatrix->ExtractMyRowView(i,NumEntries,RowValues);
for (j=0; j < NumEntries; j++) x_tmp_p[i] += std::abs(RowValues[j]);
}
TEUCHOS_TEST_FOR_EXCEPT(0!=x.Export(x_tmp, *crsMatrix->Exporter(), Add)); //Export partial row sums to x.
}
else if (crsMatrix->Graph().RowMap().SameAs(x.Map())) {
for (i=0; i < crsMatrix->NumMyRows(); i++) {
int NumEntries = 0;
double * RowValues = 0;
crsMatrix->ExtractMyRowView(i,NumEntries,RowValues);
double scale = 0.0;
for (j=0; j < NumEntries; j++) scale += std::abs(RowValues[j]);
xp[i] = scale;
}
}
else { // x.Map different than both crsMatrix->Graph().RowMap() and crsMatrix->Graph().RangeMap()
TEUCHOS_TEST_FOR_EXCEPT(true); // The map of x must be the RowMap or RangeMap of A.
}
}
int main(int argc, char *argv[])
{
// Initialize MPI
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
#endif
// Create a communicator for Epetra objects
#ifdef HAVE_MPI
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm Comm;
#endif
int * testInt = new int[100];
delete [] testInt;
bool verbose = false;
if (argc > 1)
if (argv[1][0]=='-' && argv[1][1]=='v')
verbose = true;
// Get the process ID and the total number of processors
int MyPID = Comm.MyPID();
int NumProc = Comm.NumProc();
// Set up theolver options parameter list
Teuchos::RCP<Teuchos::ParameterList> noxParamsPtr = Teuchos::rcp(new Teuchos::ParameterList);
Teuchos::ParameterList & noxParams = *(noxParamsPtr.get());
// Set up the printing utilities
// Only print output if the "-v" flag is set on the command line
Teuchos::ParameterList& printParams = noxParams.sublist("Printing");
printParams.set("MyPID", MyPID);
printParams.set("Output Precision", 5);
printParams.set("Output Processor", 0);
if( verbose )
printParams.set("Output Information",
NOX::Utils::OuterIteration +
NOX::Utils::OuterIterationStatusTest +
NOX::Utils::InnerIteration +
NOX::Utils::Parameters +
NOX::Utils::Details +
NOX::Utils::Warning +
NOX::Utils::TestDetails);
else
printParams.set("Output Information", NOX::Utils::Error +
NOX::Utils::TestDetails);
Teuchos::RCP<NOX::Utils> printing = Teuchos::rcp( new NOX::Utils(printParams) );
// Identify the test problem
if (printing->isPrintType(NOX::Utils::TestDetails))
printing->out() << "Starting epetra/NOX_Operators/NOX_BroydenOp.exe" << std::endl;
// Identify processor information
#ifdef HAVE_MPI
if (printing->isPrintType(NOX::Utils::TestDetails))
{
printing->out() << "Parallel Run" << std::endl;
printing->out() << "Number of processors = " << NumProc << std::endl;
printing->out() << "Print Process = " << MyPID << std::endl;
}
Comm.Barrier();
if (printing->isPrintType(NOX::Utils::TestDetails))
printing->out() << "Process " << MyPID << " is alive!" << std::endl;
Comm.Barrier();
#else
if (printing->isPrintType(NOX::Utils::TestDetails))
printing->out() << "Serial Run" << std::endl;
#endif
int status = 0;
// Create a TestCompare class
NOX::Epetra::TestCompare tester( printing->out(), *printing);
double abstol = 1.e-4;
double reltol = 1.e-4 ;
// Test NOX::Epetra::BroydenOperator
int numGlobalElems = 3 * NumProc;
Epetra_Map broydenRowMap ( numGlobalElems, 0, Comm );
Epetra_Vector broydenWorkVec ( broydenRowMap );
Epetra_CrsGraph broydenWorkGraph( Copy, broydenRowMap, 0 );
std::vector<int> globalIndices(3);
for( int lcol = 0; lcol < 3; ++lcol )
globalIndices[lcol] = 3 * MyPID + lcol;
std::vector<int> myGlobalIndices(2);
// Row 1 structure
myGlobalIndices[0] = globalIndices[0];
myGlobalIndices[1] = globalIndices[2];
broydenWorkGraph.InsertGlobalIndices( globalIndices[0], 2, &myGlobalIndices[0] );
// Row 2 structure
myGlobalIndices[0] = globalIndices[0];
myGlobalIndices[1] = globalIndices[1];
broydenWorkGraph.InsertGlobalIndices( globalIndices[1], 2, &myGlobalIndices[0] );
// Row 3 structure
myGlobalIndices[0] = globalIndices[1];
myGlobalIndices[1] = globalIndices[2];
broydenWorkGraph.InsertGlobalIndices( globalIndices[2], 2, &myGlobalIndices[0] );
broydenWorkGraph.FillComplete();
Teuchos::RCP<Epetra_CrsMatrix> broydenWorkMatrix =
Teuchos::rcp( new Epetra_CrsMatrix( Copy, broydenWorkGraph ) );
// Create an identity matrix
broydenWorkVec.PutScalar(1.0);
broydenWorkMatrix->ReplaceDiagonalValues(broydenWorkVec);
NOX::Epetra::BroydenOperator broydenOp( noxParams, printing, broydenWorkVec, broydenWorkMatrix, true );
broydenWorkVec[0] = 1.0;
broydenWorkVec[1] = -1.0;
broydenWorkVec[2] = 2.0;
broydenOp.setStepVector( broydenWorkVec );
broydenWorkVec[0] = 2.0;
broydenWorkVec[1] = 1.0;
broydenWorkVec[2] = 3.0;
broydenOp.setYieldVector( broydenWorkVec );
broydenOp.computeSparseBroydenUpdate();
// Create the gold matrix for comparison
Teuchos::RCP<Epetra_CrsMatrix> goldMatrix = Teuchos::rcp( new Epetra_CrsMatrix( Copy, broydenWorkGraph ) );
int numCols ;
double * values ;
// Row 1 answers
goldMatrix->ExtractMyRowView( 0, numCols, values );
values[0] = 6.0 ;
values[1] = 2.0 ;
// Row 2 answers
goldMatrix->ExtractMyRowView( 1, numCols, values );
values[0] = 5.0 ;
values[1] = 0.0 ;
// Row 3 structure
goldMatrix->ExtractMyRowView( 2, numCols, values );
values[0] = -1.0 ;
values[1] = 7.0 ;
goldMatrix->Scale(0.2);
status += tester.testCrsMatrices( broydenOp.getBroydenMatrix(), *goldMatrix, reltol, abstol,
"Broyden Sparse Operator Update Test" );
// Now try a dense Broyden Update
Epetra_CrsGraph broydenWorkGraph2( Copy, broydenRowMap, 0 );
myGlobalIndices.resize(3);
// All Rowsstructure
myGlobalIndices[0] = globalIndices[0];
myGlobalIndices[1] = globalIndices[1];
myGlobalIndices[2] = globalIndices[2];
broydenWorkGraph2.InsertGlobalIndices( globalIndices[0], 3, &myGlobalIndices[0] );
broydenWorkGraph2.InsertGlobalIndices( globalIndices[1], 3, &myGlobalIndices[0] );
broydenWorkGraph2.InsertGlobalIndices( globalIndices[2], 3, &myGlobalIndices[0] );
broydenWorkGraph2.FillComplete();
Teuchos::RCP<Epetra_CrsMatrix> broydenWorkMatrix2 = Teuchos::rcp( new Epetra_CrsMatrix( Copy, broydenWorkGraph2 ) );
// Create an identity matrix
broydenWorkVec.PutScalar(1.0);
broydenWorkMatrix2->ReplaceDiagonalValues(broydenWorkVec);
NOX::Epetra::BroydenOperator broydenOp2( noxParams, printing, broydenWorkVec, broydenWorkMatrix2, true );
broydenWorkVec[0] = 1.0;
broydenWorkVec[1] = -1.0;
broydenWorkVec[2] = 2.0;
broydenOp2.setStepVector( broydenWorkVec );
broydenWorkVec[0] = 2.0;
broydenWorkVec[1] = 1.0;
broydenWorkVec[2] = 3.0;
broydenOp2.setYieldVector( broydenWorkVec );
broydenOp2.computeSparseBroydenUpdate();
// Create the gold matrix for comparison
Teuchos::RCP<Epetra_CrsMatrix> goldMatrix2 = Teuchos::rcp( new Epetra_CrsMatrix( Copy, broydenWorkGraph2 ) );
// Row 1 answers
goldMatrix2->ExtractMyRowView( 0, numCols, values );
values[0] = 7.0 ;
values[1] = -1.0 ;
values[2] = 2.0 ;
// Row 2 answers
goldMatrix2->ExtractMyRowView( 1, numCols, values );
values[0] = 2.0 ;
values[1] = 4.0 ;
values[2] = 4.0 ;
// Row 3 structure
goldMatrix2->ExtractMyRowView( 2, numCols, values );
values[0] = 1.0 ;
values[1] = -1.0 ;
values[2] = 8.0 ;
double scaleF = 1.0 / 6.0;
goldMatrix2->Scale( scaleF );
status += tester.testCrsMatrices( broydenOp2.getBroydenMatrix(), *goldMatrix2, reltol, abstol,
"Broyden Sparse Operator Update Test (Dense)" );
// Now test the ability to remove active entries in the Broyden update
Epetra_CrsGraph inactiveGraph( Copy, broydenRowMap, 0 );
// Row 1 structure
inactiveGraph.InsertGlobalIndices( globalIndices[0], 1, &myGlobalIndices[1] );
// Row 2 structure
inactiveGraph.InsertGlobalIndices( globalIndices[1], 1, &myGlobalIndices[2] );
// Row 3 structure
inactiveGraph.InsertGlobalIndices( globalIndices[2], 1, &myGlobalIndices[0] );
inactiveGraph.FillComplete();
// Inactivate entries in dense matrix to arrive again at the original sparse structure
broydenOp2.removeEntriesFromBroydenUpdate( inactiveGraph );
#ifdef HAVE_NOX_DEBUG
if( verbose )
broydenOp2.outputActiveEntries();
#endif
// Reset to the identity matrix
broydenOp2.resetBroydenMatrix( *broydenWorkMatrix2 );
// Step and Yield vectors are already set
broydenOp2.computeSparseBroydenUpdate();
status += tester.testCrsMatrices( broydenOp2.getBroydenMatrix(), *goldMatrix, reltol, abstol,
"Broyden Sparse Operator Update Test (Entry Removal)", false );
// Summarize test results
if( status == 0 )
printing->out() << "Test passed!" << std::endl;
else
printing->out() << "Test failed!" << std::endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
// Final return value (0 = successfull, non-zero = failure)
return status;
}
int TestOneMatrix( std::string HBname, std::string MMname, std::string TRIname, Epetra_Comm &Comm, bool verbose ) {
if ( Comm.MyPID() != 0 ) verbose = false ;
Epetra_Map * readMap = 0;
Epetra_CrsMatrix * HbA = 0;
Epetra_Vector * Hbx = 0;
Epetra_Vector * Hbb = 0;
Epetra_Vector * Hbxexact = 0;
Epetra_CrsMatrix * TriplesA = 0;
Epetra_Vector * Triplesx = 0;
Epetra_Vector * Triplesb = 0;
Epetra_Vector * Triplesxexact = 0;
Epetra_CrsMatrix * MatrixMarketA = 0;
Epetra_Vector * MatrixMarketx = 0;
Epetra_Vector * MatrixMarketb = 0;
Epetra_Vector * MatrixMarketxexact = 0;
int TRI_Size = TRIname.size() ;
std::string LastFiveBytes = TRIname.substr( EPETRA_MAX(0,TRI_Size-5), TRI_Size );
if ( LastFiveBytes == ".TimD" ) {
// Call routine to read in a file with a Tim Davis header and zero-based indexing
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra64( &TRIname[0], false, Comm,
readMap, TriplesA, Triplesx,
Triplesb, Triplesxexact, false, true, true ) );
delete readMap;
} else {
if ( LastFiveBytes == ".triU" ) {
// Call routine to read in unsymmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra64( &TRIname[0], false, Comm,
readMap, TriplesA, Triplesx,
Triplesb, Triplesxexact, false, false ) );
delete readMap;
} else {
if ( LastFiveBytes == ".triS" ) {
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra64( &TRIname[0], true, Comm,
readMap, TriplesA, Triplesx,
Triplesb, Triplesxexact, false, false ) );
delete readMap;
} else {
assert( false ) ;
}
}
}
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra64( &MMname[0], Comm, readMap,
MatrixMarketA, MatrixMarketx,
MatrixMarketb, MatrixMarketxexact) );
delete readMap;
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra64( &HBname[0], Comm, readMap, HbA, Hbx,
Hbb, Hbxexact) ;
#if 0
std::cout << " HbA " ;
HbA->Print( std::cout ) ;
std::cout << std::endl ;
std::cout << " MatrixMarketA " ;
MatrixMarketA->Print( std::cout ) ;
std::cout << std::endl ;
std::cout << " TriplesA " ;
TriplesA->Print( std::cout ) ;
std::cout << std::endl ;
#endif
int TripleErr = 0 ;
int MMerr = 0 ;
for ( int i = 0 ; i < 10 ; i++ )
{
double resid_Hb_Triples;
double resid_Hb_Matrix_Market;
double norm_A ;
Hbx->Random();
//
// Set the output vectors to different values:
//
Triplesb->PutScalar(1.1);
Hbb->PutScalar(1.2);
MatrixMarketb->PutScalar(1.3);
HbA->Multiply( false, *Hbx, *Hbb );
norm_A = HbA->NormOne( ) ;
TriplesA->Multiply( false, *Hbx, *Triplesb );
Triplesb->Update( 1.0, *Hbb, -1.0 ) ;
MatrixMarketA->Multiply( false, *Hbx, *MatrixMarketb );
MatrixMarketb->Update( 1.0, *Hbb, -1.0 ) ;
Triplesb->Norm1( &resid_Hb_Triples ) ;
MatrixMarketb->Norm1( &resid_Hb_Matrix_Market ) ;
TripleErr += ( resid_Hb_Triples > 1e-11 * norm_A ) ;
MMerr += ( resid_Hb_Matrix_Market > 1e-11 * norm_A ) ;
if ( verbose && resid_Hb_Triples > 1e-11 * norm_A )
std::cout << " resid_Hb_Triples = " << resid_Hb_Triples
<< " norm_A = " << norm_A << std::endl ;
if ( verbose && resid_Hb_Matrix_Market > 1e-11 * norm_A )
std::cout << " resid_Hb_Matrix_Market = " << resid_Hb_Matrix_Market
<< " norm_A = " << norm_A << std::endl ;
}
if ( verbose ) {
if ( TripleErr ) std::cout << " Error in reading " << HBname << " or " << TRIname << std::endl ;
if ( MMerr ) std::cout << " Error in reading " << HBname << " or " << MMname << std::endl ;
}
delete HbA;
delete Hbx;
delete Hbb;
delete Hbxexact;
delete TriplesA;
delete Triplesx;
delete Triplesb;
delete Triplesxexact;
delete MatrixMarketA;
delete MatrixMarketx;
delete MatrixMarketb;
delete MatrixMarketxexact;
delete readMap;
return TripleErr+MMerr ;
}
//! Initialize vector
static void init(Epetra_Vector& vec, double val) { vec.PutScalar(val); }
void
exampleRoutine (const Epetra_Comm& comm,
std::ostream& out)
{
using std::endl;
// Print out the Epetra software version.
if (comm.MyPID () == 0) {
out << Epetra_Version () << endl << endl;
}
// The type of global indices. You could just set this to int,
// but we want the example to work for Epetra64 as well.
#ifdef EPETRA_NO_32BIT_GLOBAL_INDICES
// Epetra was compiled only with 64-bit global index support, so use
// 64-bit global indices.
typedef long long global_ordinal_type;
#else
// Epetra was compiled with 32-bit global index support. If
// EPETRA_NO_64BIT_GLOBAL_INDICES is defined, it does not also
// support 64-bit indices.
typedef int global_ordinal_type;
#endif // EPETRA_NO_32BIT_GLOBAL_INDICES
//////////////////////////////////////////////////////////////////////
// Create some Epetra_Map objects
//////////////////////////////////////////////////////////////////////
//
// Epetra has local and global Maps. Local maps describe objects
// that are replicated over all participating MPI processes. Global
// maps describe distributed objects. You can do imports and
// exports between local and global maps; this is how you would turn
// locally replicated objects into distributed objects and vice
// versa.
//
// The total (global, i.e., over all MPI processes) number of
// entries in the Map. This has the same type as that of global
// indices, so it can represent very large values if Epetra was
// built with 64-bit global index support.
//
// For this example, we scale the global number of entries in the
// Map with the number of MPI processes. That way, you can run this
// example with any number of MPI processes and every process will
// still have a positive number of entries.
const global_ordinal_type numGlobalEntries = comm.NumProc () * 5;
// Tpetra can index the entries of a Map starting with 0 (C style),
// 1 (Fortran style), or any base you want. 1-based indexing is
// handy when interfacing with Fortran. We choose 0-based indexing
// here. This also has the same type as that of global indices.
const global_ordinal_type indexBase = 0;
// Construct a Map that puts the same number of equations on each
// (MPI) process. The Epetra_Comm is passed in by value, but that's
// OK, because Epetra_Comm has shallow copy semantics. (Its copy
// constructor and assignment operator do not call MPI_Comm_dup;
// they just pass along the MPI_Comm.)
Epetra_Map contigMap (numGlobalEntries, indexBase, comm);
// contigMap is contiguous by construction.
if (! contigMap.LinearMap ()) {
throw std::logic_error ("The supposedly contiguous Map isn't contiguous.");
}
// Let's create a second Map. It will have the same number of
// global entries per process, but will distribute them differently,
// in round-robin (1-D cyclic) fashion instead of contiguously.
// We'll use the version of the Map constructor that takes, on each
// MPI process, a list of the global indices in the Map belonging to
// that process. You can use this constructor to construct an
// overlapping (also called "not 1-to-1") Map, in which one or more
// entries are owned by multiple processes. We don't do that here;
// we make a nonoverlapping (also called "1-to-1") Map.
const int numGblIndsPerProc = 5;
global_ordinal_type* gblIndList = new global_ordinal_type [numGblIndsPerProc];
const int numProcs = comm.NumProc ();
const int myRank = comm.MyPID ();
for (int k = 0; k < numGblIndsPerProc; ++k) {
gblIndList[k] = myRank + k*numProcs;
}
Epetra_Map cyclicMap (numGlobalEntries, numGblIndsPerProc,
gblIndList, indexBase, comm);
// The above constructor makes a deep copy of the input index list,
// so it's safe to deallocate that list after this constructor
// completes.
if (gblIndList != NULL) {
delete [] gblIndList;
gblIndList = NULL;
}
// If there's more than one MPI process in the communicator,
// then cyclicMap is definitely NOT contiguous.
if (comm.NumProc () > 1 && cyclicMap.LinearMap ()) {
throw std::logic_error ("The cyclic Map claims to be contiguous.");
}
// contigMap and cyclicMap should always be compatible. However, if
// the communicator contains more than 1 process, then contigMap and
// cyclicMap are NOT the same.
// if (! contigMap.isCompatible (*cyclicMap)) {
// throw std::logic_error ("contigMap should be compatible with cyclicMap, "
// "but it's not.");
// }
if (comm.NumProc () > 1 && contigMap.SameAs (cyclicMap)) {
throw std::logic_error ("contigMap should not be the same as cyclicMap.");
}
//////////////////////////////////////////////////////////////////////
// We have maps now, so we can create vectors.
//////////////////////////////////////////////////////////////////////
// Create an Epetra_Vector with the contiguous Map we created above.
// This version of the constructor will fill the vector with zeros.
// The Vector constructor takes a Map by value, but that's OK,
// because Epetra_Map has shallow copy semantics. It uses reference
// counting internally to avoid copying data unnecessarily.
Epetra_Vector x (contigMap);
// The copy constructor performs a deep copy.
// x and y have the same Map.
Epetra_Vector y (x);
// Create a Vector with the 1-D cyclic Map. Calling the constructor
// with false for the second argument leaves the data uninitialized,
// so that you can fill it later without paying the cost of
// initially filling it with zeros.
Epetra_Vector z (cyclicMap, false);
// Set the entries of z to (pseudo)random numbers. Please don't
// consider this a good parallel pseudorandom number generator.
(void) z.Random ();
// Set the entries of x to all ones.
(void) x.PutScalar (1.0);
// Define some constants for use below.
const double alpha = 3.14159;
const double beta = 2.71828;
const double gamma = -10.0;
// x = beta*x + alpha*z
//
// This is a legal operation! Even though the Maps of x and z are
// not the same, their Maps are compatible. Whether it makes sense
// or not depends on your application.
(void) x.Update (alpha, z, beta);
(void) y.PutScalar (42.0); // Set all entries of y to 42.0
// y = gamma*y + alpha*x + beta*z
y.Update (alpha, x, beta, z, gamma);
// Compute the 2-norm of y.
//
// The norm may have a different type than scalar_type.
// For example, if scalar_type is complex, then the norm is real.
// The ScalarTraits "traits class" gives us the type of the norm.
double theNorm = 0.0;
(void) y.Norm2 (&theNorm);
// Print the norm of y on Proc 0.
out << "Norm of y: " << theNorm << endl;
}
void
HMX_PDE::computeSourceTerm(map<string, Epetra_Vector*> fields,
Epetra_Vector& result)
{
// All dependent variables should be in place before now
// We assume all other registered dependent problems are species which
// affect the reaction rate of this specie
Epetra_Vector *TvecPtr = 0;
map<string, Epetra_Vector*>::iterator iter = fields.find(tempFieldName);
if( iter == fields.end() )
{
std::cout << "ERROR: Cannot find Temperature field \"" << tempFieldName
<< "\" for use in computeSourceTerm for problem \""
<< getName() << std::endl;
throw "HMX_PDE ERROR";
}
else
TvecPtr = (*iter).second;
Epetra_Vector &T = *TvecPtr;
// If this problem is the temperature equation, don't compute a source
// term. This would be where a volumetric heating term would go.
if( getName() == tempFieldName )
{
result.PutScalar(0.0);
return;
}
else
{
double rateK;
map<string, double>::iterator requiredFieldIter;
map<string, double>::iterator requiredFieldEnd = SrcTermExponent.end();
for( int i = 0; i<result.MyLength(); i++ ) {
rateK = pow(T[i],StericCoef) * PreExp_A *
exp( -ActEnergy / (Const_R * T[i]) );
result[i] = 1.0; // Start point for product
// Loop over all required fields and contribute to product
for( requiredFieldIter = SrcTermExponent.begin();
requiredFieldIter != requiredFieldEnd; requiredFieldIter++)
{
iter = fields.find( (*requiredFieldIter).first );
if( iter == fields.end() )
{
std::cout << "ERROR: Cannot find required field \""
<< (*requiredFieldIter).first
<< "\" for use in computeSourceTerm for problem \""
<< getName() << std::endl;
throw "HMX_PDE ERROR";
}
Epetra_Vector &reqFieldVec = *((*iter).second);
result[i] *= pow( reqFieldVec[i], (*requiredFieldIter).second );
}
result[i] *= rateK;
}
}
}
int test_azoo_scaling(Epetra_CrsMatrix& A,
Epetra_Vector& x,
Epetra_Vector& b,
bool verbose)
{
Epetra_Vector vec1(x);
Epetra_Vector vec2(x);
Epetra_Vector diag(x);
Epetra_Vector vec3(x);
Epetra_Vector vec4(x);
Epetra_Vector rhs(x);
Epetra_Vector soln_none(x);
Epetra_Vector soln_jacobi(x);
Epetra_Vector soln_rowsum(x);
Epetra_Vector soln_symdiag(x);
vec1.PutScalar(1.0);
A.Multiply(false, vec1, vec2);
A.ExtractDiagonalCopy(diag);
double* diag_vals = NULL;
diag.ExtractView(&diag_vals);
int* options = new int[AZ_OPTIONS_SIZE];
double* params = new double[AZ_PARAMS_SIZE];
AZ_defaults(options, params);
options[AZ_output] = verbose ? 1 : AZ_none;
options[AZ_scaling] = AZ_Jacobi;
AztecOO::MatrixData mdata(&A);
AZ_MATRIX* Amat = AZ_matrix_create(vec1.Map().NumMyElements());
AZ_set_MATFREE(Amat, (void*)(&mdata), Epetra_Aztec_matvec);
AZ_SCALING* scaling = AZ_scaling_create();
double* xvals = NULL, *bvals = NULL;
x.ExtractView(&xvals);
b.ExtractView(&bvals);
int err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cout << "AztecOO_scale_epetra returned err="<<err<<endl;
}
return(err);
}
A.Multiply(false, vec1, vec3);
vec4.Multiply(1.0, diag, vec3, 0.0);
double vec2nrm, vec4nrm;
vec2.Norm2(&vec2nrm);
vec4.Norm2(&vec4nrm);
if (fabs(vec2nrm - vec4nrm) > 1.e-6) {
return(-1);
}
//now call the scaling function again, just to allow for
//testing memory-leak issues.
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cout << "AztecOO_scale_epetra returned err="<<err<<endl;
}
return(err);
}
AztecOO_scale_epetra(AZ_DESTROY_SCALING_DATA, Amat, options,
bvals, xvals, NULL, scaling);
x.PutScalar(1.0);
Epetra_CrsMatrix* Atmp = create_and_fill_crs_matrix(A.RowMap());
Atmp->Multiply(false, x, rhs);
x.PutScalar(0.0);
AztecOO azoo(&A, &x, &b);
azoo.SetAztecOption(AZ_scaling, AZ_Jacobi);
if (verbose) {
azoo.SetAztecOption(AZ_output, 1);
}
else {
azoo.SetAztecOption(AZ_output, AZ_none);
}
azoo.Iterate(100, 1.e-6);
delete Atmp;
Epetra_CrsMatrix* Atmp1 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp1->Multiply(false, x, rhs);
soln_rowsum.PutScalar(0.0);
AztecOO azoo1(Atmp1, &soln_rowsum, &rhs);
azoo1.SetAztecOption(AZ_scaling, AZ_row_sum);
azoo1.Iterate(100, 1.e-8);
delete Atmp1;
Epetra_CrsMatrix* Atmp2 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp2->Multiply(false, x, rhs);
soln_symdiag.PutScalar(0.0);
AztecOO azoo2(Atmp2, &soln_symdiag, &rhs);
azoo2.SetAztecOption(AZ_scaling, AZ_sym_diag);
azoo2.Iterate(100, 1.e-8);
delete Atmp2;
Epetra_CrsMatrix* Atmp3 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp3->Multiply(false, x, rhs);
soln_none.PutScalar(0.0);
AztecOO azoo3(Atmp3, &soln_none, &rhs);
azoo3.SetAztecOption(AZ_scaling, AZ_none);
azoo3.Iterate(100, 1.e-8);
delete Atmp3;
Epetra_CrsMatrix* Atmp4 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp4->Multiply(false, x, rhs);
soln_jacobi.PutScalar(0.0);
AztecOO azoo4(Atmp4, &soln_jacobi, &rhs);
azoo4.SetAztecOption(AZ_scaling, AZ_Jacobi);
azoo4.Iterate(100, 1.e-8);
delete Atmp4;
//at this point, soln_none, soln_jacobi, soln_rowsum and soln_symdiag
//should be the same or at least close to the same, since the
//matrix used in the solution has well-behaved coefficients.
//form vec1 = soln_none - soln_rowsum
vec1.PutScalar(0.0);
vec1.Update(1.0, soln_none, 0.0);
vec1.Update(-1.0, soln_rowsum, 1.0);
double norm_check1= 0.0;
vec1.Norm2(&norm_check1);
//form vec2 = soln_none - soln_symdiag
vec2.PutScalar(0.0);
vec2.Update(1.0, soln_none, 0.0);
vec2.Update(-1.0, soln_symdiag, 1.0);
double norm_check2= 0.0;
vec2.Norm2(&norm_check2);
//form vec3 = soln_none - soln_jacobi
vec3.PutScalar(0.0);
vec3.Update(1.0, soln_none, 0.0);
vec3.Update(-1.0, soln_jacobi, 1.0);
double norm_check3= 0.0;
vec3.Norm2(&norm_check3);
if (std::abs(norm_check1) > 1.e-6) {
if (verbose) {
cerr << "AZ_row_sum scaling produced bad soln"
<< endl;
}
return(-1);
}
if (std::abs(norm_check2) > 1.e-6) {
if (verbose) {
cerr << "AZ_sym_diag scaling produced bad soln"
<< endl;
}
return(-1);
}
if (std::abs(norm_check3) > 1.e-6) {
if (verbose) {
cerr << "AZ_Jacobi scaling produced bad soln"
<< endl;
}
return(-1);
}
options[AZ_pre_calc] = AZ_reuse;
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err == 0) {
if (verbose) {
cerr << "AztecOO_scale_epetra failed to return err when"
<< " asked to reuse non-existent scaling data."<<endl;
}
return(-1);
}
options[AZ_keep_info] = 1;
options[AZ_pre_calc] = AZ_calc;
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cerr << "AztecOO_scale_epetra returned err=="<<err<<endl;
}
return(err);
}
options[AZ_keep_info] = 0;
options[AZ_pre_calc] = AZ_reuse;
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cerr << "AztecOO_scale_epetra returned err=="<<err
<<" when asked to reuse scaling data"<<endl;
}
return(err);
}
options[AZ_pre_calc] = AZ_calc;
err = AztecOO_scale_epetra(AZ_DESTROY_SCALING_DATA, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
std::cerr << "AztecOO_scale_epetra returned err=="<<err
<< " when asked to destroy scaling data."<<std::endl;
}
return(err);
}
AZ_matrix_destroy(&Amat);
delete [] options;
delete [] params;
AZ_scaling_destroy(&scaling);
AZ_manage_memory(0, AZ_CLEAR_ALL, 0, 0, 0);
return(0);
}
int test_azoo_with_ilut(Epetra_CrsMatrix& A,
Epetra_Vector& x,
Epetra_Vector& b,
bool verbose)
{
x.PutScalar(0.0);
AztecOO* azoo0 = new AztecOO(&A, &x, &b);
azoo0->SetAztecOption(AZ_solver, AZ_gmres);
azoo0->SetAztecOption(AZ_output, AZ_none);
azoo0->SetAztecOption(AZ_conv, AZ_Anorm);
azoo0->SetAztecOption(AZ_precond, AZ_dom_decomp);
azoo0->SetAztecOption(AZ_subdomain_solve, AZ_ilut);
azoo0->SetAztecOption(AZ_keep_info, 1);
if (verbose) {
cout << "testing AztecOO with GMRES and ILUT, AZ_keep_info==1" << endl;
}
int maxiters = 100;
double tolerance = 1.e-12;
int err = azoo0->Iterate(maxiters, tolerance);
if (err != 0) {
if (verbose) cout << "AztecOO::Iterate return err="<<err<<endl;
return(err);
}
double resid = resid2norm(A, x, b);
if (verbose) {
cout << "residual 2-norm after GMRES/ILUT solve: "
<< resid << endl;
}
if (resid > 1.e-6) {
return(-1);
}
if (verbose) {
cout << "solving with GMRES/ILUT again, AZ_pre_calc==AZ_reuse"
<< endl << "(will error out if factors weren't kept from"
<< " previous solve)"<<endl;
}
azoo0->SetAztecOption(AZ_pre_calc, AZ_reuse);
x.PutScalar(0.0);
err = azoo0->Iterate(maxiters, tolerance);
if (err != 0) {
if (verbose) cout << "AztecOO::Iterate return err="<<err<<endl;
return(err);
}
double resid2 = resid2norm(A, x, b);
if (verbose) {
cout << "after second GMRES/ILUT solve, residual 2-norm: "
<< resid2 <<endl;
}
if (resid2 > 1.e-6) {
return(-1);
}
if (verbose) {
cout << "azoo0->SolveTime(): " << azoo0->SolveTime() << endl;
}
delete azoo0;
return(0);
}
int test_azoo_as_precond_op(Epetra_CrsMatrix& A,
Epetra_Vector& x,
Epetra_Vector& b,
bool verbose)
{
AztecOO* azoo0 = new AztecOO(&A, &x, &b);
azoo0->SetAztecOption(AZ_solver, AZ_cg);
azoo0->SetAztecOption(AZ_output, AZ_none);
azoo0->SetAztecOption(AZ_conv, AZ_none);
AztecOO_Operator azooOp(azoo0, 10);
Epetra_LinearProblem elp(&A, &x, &b);
x.PutScalar(0.0);
AztecOO* azoo1 = new AztecOO(elp);
azoo1->SetPrecOperator(&azooOp);
azoo1->SetAztecOption(AZ_solver, AZ_gmres);
azoo1->SetAztecOption(AZ_output, AZ_none);
azoo1->SetAztecOption(AZ_conv, AZ_none);
if (verbose) {
cout << "testing recursive solve (AztecOO as"
<< " preconditioner for another AztecOO)."<<endl;
}
int maxiters = 100;
double tolerance = 1.e-12;
azoo1->Iterate(maxiters, tolerance);
double resid1 = resid2norm(A, x, b);
if (verbose) {
cout << "residual 2-norm after recursive solve: "
<< resid1 << endl;
}
if (resid1 > 1.e-6) {
return(-1);
}
if (verbose) {
cout << "now make sure the precond AztecOO instance"
<< " hasn't been corrupted."<<endl;
}
// AZ_manage_memory(0, -43, 0, 0, 0);
x.PutScalar(0.0);
azoo0->Iterate(&A, &x, &b, maxiters, tolerance);
double resid0 = resid2norm(A, x, b);
if (verbose) {
cout << "residual 2-norm: " << resid0 << endl;
}
if (resid0 > 1.e-6) {
return(-1);
}
delete azoo0;
delete azoo1;
return(0);
}
int
main (int argc, char *argv[])
{
using Teuchos::ArrayRCP;
using Teuchos::ArrayView;
using Teuchos::Comm;
using Teuchos::CommandLineProcessor;
using Teuchos::FancyOStream;
using Teuchos::getFancyOStream;
using Teuchos::OSTab;
using Teuchos::ptr;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcpFromRef;
using std::cout;
using std::endl;
bool success = true; // May be changed by tests
Teuchos::oblackholestream blackHole;
//Teuchos::GlobalMPISession (&argc, &argv, &blackHole);
MPI_Init (&argc, &argv);
//
// Construct communicators, and verify that we are on 4 processors.
//
// Construct a Teuchos Comm object.
RCP<const Comm<int> > teuchosComm = Teuchos::DefaultComm<int>::getComm();
const int numProcs = teuchosComm->getSize();
const int pid = teuchosComm->getRank();
RCP<FancyOStream> pOut =
getFancyOStream (rcpFromRef ((pid == 0) ? std::cout : blackHole));
FancyOStream& out = *pOut;
// Verify that we are on four processors (which manifests the bug).
if (teuchosComm->getSize() != 4) {
out << "This test must be run on four processors. Exiting ..." << endl;
return EXIT_FAILURE;
}
// We also need an Epetra Comm, so that we can compare Tpetra and
// Epetra results.
Epetra_MpiComm epetraComm (MPI_COMM_WORLD);
//
// Default values of command-line options.
//
bool verbose = false;
bool printEpetra = false;
bool printTpetra = false;
CommandLineProcessor cmdp (false,true);
//
// Set command-line options.
//
cmdp.setOption ("verbose", "quiet", &verbose, "Print verbose output.");
// Epetra and Tpetra output will ask the Maps and Import objects to
// print themselves in distributed, maximally verbose fashion. It's
// best to turn on either Epetra or Tpetra, but not both. Then you
// can compare their output side by side.
cmdp.setOption ("printEpetra", "dontPrintEpetra", &printEpetra,
"Print Epetra output (in verbose mode only).");
cmdp.setOption ("printTpetra", "dontPrintTpetra", &printTpetra,
"Print Tpetra output (in verbose mode only).");
// Parse command-line options.
if (cmdp.parse (argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
out << "End Result: TEST FAILED" << endl;
MPI_Finalize ();
return EXIT_FAILURE;
}
if (verbose) {
out << "Running test on " << numProcs << " process"
<< (numProcs != 1 ? "es" : "") << "." << endl;
}
// The maps for this problem are derived from a 3D structured mesh.
// In this example, the dimensions are 4x4x2 and there are 2
// processors assigned to the first dimension and 2 processors
// assigned to the second dimension, with no parallel decomposition
// along the third dimension. The "owned" arrays represent the
// one-to-one map, with each array representing a 2x2x2 slice. If
// DIMENSIONS == 2, then only the first 4 values will be used,
// representing a 2x2(x1) slice.
int owned0[8] = { 0, 1, 4, 5,16,17,20,21};
int owned1[8] = { 2, 3, 6, 7,18,19,22,23};
int owned2[8] = { 8, 9,12,13,24,25,28,29};
int owned3[8] = {10,11,14,15,26,27,30,31};
// The "overlap" arrays represent the map with communication
// elements, with each array representing a 3x3x2 slice. If
// DIMENSIONS == 2, then only the first 9 values will be used,
// representing a 3x3(x1) slice.
int overlap0[18] = {0,1,2,4, 5, 6, 8, 9,10,16,17,18,20,21,22,24,25,26};
int overlap1[18] = {1,2,3,5, 6, 7, 9,10,11,17,18,19,21,22,23,25,26,27};
int overlap2[18] = {4,5,6,8, 9,10,12,13,14,20,21,22,24,25,26,28,29,30};
int overlap3[18] = {5,6,7,9,10,11,13,14,15,21,22,23,25,26,27,29,30,31};
// Construct the owned and overlap maps for both Epetra and Tpetra.
int* owned;
int* overlap;
if (pid == 0) {
owned = owned0;
overlap = overlap0;
}
else if (pid == 1) {
owned = owned1;
overlap = overlap1;
}
else if (pid == 2) {
owned = owned2;
overlap = overlap2;
}
else {
owned = owned3;
overlap = overlap3;
}
#if DIMENSIONS == 2
int ownedSize = 4;
int overlapSize = 9;
#elif DIMENSIONS == 3
int ownedSize = 8;
int overlapSize = 18;
#endif
// Create the two Epetra Maps. Source for the Import is the owned
// map; target for the Import is the overlap map.
Epetra_Map epetraOwnedMap ( -1, ownedSize, owned, 0, epetraComm);
Epetra_Map epetraOverlapMap (-1, overlapSize, overlap, 0, epetraComm);
if (verbose && printEpetra) {
// Have the Epetra_Map objects describe themselves.
//
// Epetra_BlockMap::Print() takes an std::ostream&, and expects
// all MPI processes to be able to write to it. (The method
// handles its own synchronization.)
out << "Epetra owned map:" << endl;
epetraOwnedMap.Print (std::cout);
out << "Epetra overlap map:" << endl;
epetraOverlapMap.Print (std::cout);
}
// Create the two Tpetra Maps. The "invalid" global element count
// input tells Tpetra::Map to compute the global number of elements
// itself.
const int invalid = Teuchos::OrdinalTraits<int>::invalid();
RCP<Tpetra::Map<int> > tpetraOwnedMap =
rcp (new Tpetra::Map<int> (invalid, ArrayView<int> (owned, ownedSize),
0, teuchosComm));
tpetraOwnedMap->setObjectLabel ("Owned Map");
RCP<Tpetra::Map<int> > tpetraOverlapMap =
rcp (new Tpetra::Map<int> (invalid, ArrayView<int> (overlap, overlapSize),
0, teuchosComm));
tpetraOverlapMap->setObjectLabel ("Overlap Map");
// In verbose mode, have the Tpetra::Map objects describe themselves.
if (verbose && printTpetra) {
Teuchos::EVerbosityLevel verb = Teuchos::VERB_EXTREME;
// Tpetra::Map::describe() takes a FancyOStream, but expects all
// MPI processes to be able to write to it. (The method handles
// its own synchronization.)
RCP<FancyOStream> globalOut = getFancyOStream (rcpFromRef (std::cout));
out << "Tpetra owned map:" << endl;
{
OSTab tab (globalOut);
tpetraOwnedMap->describe (*globalOut, verb);
}
out << "Tpetra overlap map:" << endl;
{
OSTab tab (globalOut);
tpetraOverlapMap->describe (*globalOut, verb);
}
}
// Use the owned and overlap maps to construct an importer for both
// Epetra and Tpetra.
Epetra_Import epetraImporter (epetraOverlapMap, epetraOwnedMap );
Tpetra::Import<int> tpetraImporter (tpetraOwnedMap , tpetraOverlapMap);
// In verbose mode, have the Epetra_Import object describe itself.
if (verbose && printEpetra) {
out << "Epetra importer:" << endl;
// The importer's Print() method takes an std::ostream& and plans
// to write to it on all MPI processes (handling synchronization
// itself).
epetraImporter.Print (std::cout);
out << endl;
}
// In verbose mode, have the Tpetra::Import object describe itself.
if (verbose && printTpetra) {
out << "Tpetra importer:" << endl;
// The importer doesn't implement Teuchos::Describable. It wants
// std::cout and plans to write to it on all MPI processes (with
// its own synchronization).
tpetraImporter.print (std::cout);
out << endl;
}
// Construct owned and overlap vectors for both Epetra and Tpetra.
Epetra_Vector epetraOwnedVector (epetraOwnedMap );
Epetra_Vector epetraOverlapVector (epetraOverlapMap);
Tpetra::Vector<double,int> tpetraOwnedVector (tpetraOwnedMap );
Tpetra::Vector<double,int> tpetraOverlapVector (tpetraOverlapMap);
// The test is as follows: initialize the owned and overlap vectors
// with global IDs in the owned regions. Initialize the overlap
// vectors to equal -1 in the overlap regions. Then perform a
// communication from the owned vectors to the overlap vectors. The
// resulting overlap vectors should have global IDs everywhere and
// all of the -1 values should be overwritten.
// Initialize. We cannot assign directly to the Tpetra Vectors;
// instead, we extract nonconst views and assign to those. The
// results aren't guaranteed to be committed to the vector unless
// the views are released (by assigning Teuchos::null to them).
epetraOverlapVector.PutScalar(-1);
tpetraOverlapVector.putScalar(-1);
ArrayRCP<double> tpetraOwnedArray = tpetraOwnedVector.getDataNonConst(0);
ArrayRCP<double> tpetraOverlapArray = tpetraOverlapVector.getDataNonConst(0);
for (int owned_lid = 0;
owned_lid < tpetraOwnedMap->getNodeElementList().size();
++owned_lid) {
int gid = tpetraOwnedMap->getGlobalElement(owned_lid);
int overlap_lid = tpetraOverlapMap->getLocalElement(gid);
epetraOwnedVector[owned_lid] = gid;
epetraOverlapVector[overlap_lid] = gid;
tpetraOwnedArray[owned_lid] = gid;
tpetraOverlapArray[overlap_lid] = gid;
}
// Make sure that the changes to the Tpetra Vector were committed,
// by releasing the nonconst views.
tpetraOwnedArray = Teuchos::null;
tpetraOverlapArray = Teuchos::null;
// Test the Epetra and Tpetra Import.
if (verbose) {
out << "Testing Import from owned Map to overlap Map:" << endl << endl;
}
epetraOverlapVector.Import( epetraOwnedVector, epetraImporter, Insert);
tpetraOverlapVector.doImport(tpetraOwnedVector, tpetraImporter,
Tpetra::INSERT);
// Check the Import results.
success = countFailures (teuchosComm, epetraOwnedMap, epetraOwnedVector,
epetraOverlapMap, epetraOverlapVector,
tpetraOwnedMap, tpetraOwnedVector,
tpetraOverlapMap, tpetraOverlapVector, verbose);
const bool testOtherDirections = false;
if (testOtherDirections) {
//
// Reinitialize the Tpetra vectors and test whether Export works.
//
tpetraOverlapVector.putScalar(-1);
tpetraOwnedArray = tpetraOwnedVector.getDataNonConst(0);
tpetraOverlapArray = tpetraOverlapVector.getDataNonConst(0);
for (int owned_lid = 0;
owned_lid < tpetraOwnedMap->getNodeElementList().size();
++owned_lid)
{
int gid = tpetraOwnedMap->getGlobalElement(owned_lid);
int overlap_lid = tpetraOverlapMap->getLocalElement(gid);
tpetraOwnedArray[owned_lid] = gid;
tpetraOverlapArray[overlap_lid] = gid;
}
// Make sure that the changes to the Tpetra Vector were committed,
// by releasing the nonconst views.
tpetraOwnedArray = Teuchos::null;
tpetraOverlapArray = Teuchos::null;
// Make a Tpetra Export object, and test the export.
Tpetra::Export<int> tpetraExporter1 (tpetraOwnedMap, tpetraOverlapMap);
if (verbose) {
out << "Testing Export from owned Map to overlap Map:" << endl << endl;
}
tpetraOverlapVector.doExport (tpetraOwnedVector, tpetraExporter1,
Tpetra::INSERT);
// Check the Export results.
success = countFailures (teuchosComm, epetraOwnedMap, epetraOwnedVector,
epetraOverlapMap, epetraOverlapVector,
tpetraOwnedMap, tpetraOwnedVector,
tpetraOverlapMap, tpetraOverlapVector, verbose);
//
// Reinitialize the Tpetra vectors and see what Import in the
// other direction does.
//
tpetraOverlapVector.putScalar(-1);
tpetraOwnedArray = tpetraOwnedVector.getDataNonConst(0);
tpetraOverlapArray = tpetraOverlapVector.getDataNonConst(0);
for (int owned_lid = 0;
owned_lid < tpetraOwnedMap->getNodeElementList().size();
++owned_lid)
{
int gid = tpetraOwnedMap->getGlobalElement(owned_lid);
int overlap_lid = tpetraOverlapMap->getLocalElement(gid);
tpetraOwnedArray[owned_lid] = gid;
tpetraOverlapArray[overlap_lid] = gid;
}
// Make sure that the changes to the Tpetra Vector were committed,
// by releasing the nonconst views.
tpetraOwnedArray = Teuchos::null;
tpetraOverlapArray = Teuchos::null;
if (verbose) {
out << "Testing Import from overlap Map to owned Map:" << endl << endl;
}
Tpetra::Import<int> tpetraImporter2 (tpetraOverlapMap, tpetraOwnedMap);
tpetraOwnedVector.doImport (tpetraOverlapVector, tpetraImporter2,
Tpetra::INSERT);
// Check the Import results.
success = countFailures (teuchosComm, epetraOwnedMap, epetraOwnedVector,
epetraOverlapMap, epetraOverlapVector,
tpetraOwnedMap, tpetraOwnedVector,
tpetraOverlapMap, tpetraOverlapVector, verbose);
} // if testOtherDirections
out << "End Result: TEST " << (success ? "PASSED" : "FAILED") << endl;
MPI_Finalize ();
return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
/*----------------------------------------------------------------------*
| Constructor (public) m.gee 01/05|
| IMPORTANT: |
| No matter on which level we are here, the vector xfine is ALWAYS |
| a fine grid vector here! |
| this is the constructor for the ismatrixfree==true case
*----------------------------------------------------------------------*/
ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel(
int level, int nlevel, int printlevel, ML* ml,
ML_Aggregate* ag,Epetra_CrsMatrix** P,
ML_NOX::Ml_Nox_Fineinterface& interface,
const Epetra_Comm& comm, const Epetra_Vector& xfine,
bool ismatrixfree, bool isnlnCG, int nitersCG, bool broyden,
string fsmoothertype, string smoothertype, string coarsesolvetype,
int nsmooth_fine, int nsmooth, int nsmooth_coarse,
double conv_normF,
double conv_nupdate, int conv_maxiter,
int numPDE, int nullspdim, Epetra_CrsMatrix* Mat,
ML_NOX::Nox_CoarseProblem_Interface* coarseinterface)
: fineinterface_(interface),
comm_(comm)
{
level_ = level; // this level
nlevel_ = nlevel; // number of total levels
ml_printlevel_ = printlevel; // printlevel
ml_ = ml; // the global ML object
ag_ = ag; // the global ML_Aggregate object
thislevel_prec_ = 0; // this level's linear preconditioner
thislevel_ml_ = 0; // this level's local ML object
thislevel_ag_ = 0; // this level's local ML_Aggregate object
coarseinterface_ = coarseinterface; // this level's coarse interface
coarseprepost_ = 0;
xthis_ = 0; // this level's current solution matching this level's map!!!!
thislevel_A_ = 0; // this level's NOX Matrixfree operator
SmootherA_ = 0; // this level's Epetra_CrsMatrix for thislevel_prec_
ismatrixfree_ = ismatrixfree; // matrixfree flag
conv_normF_ = conv_normF; // NOX convergence test stuff
conv_nupdate_ = conv_nupdate;
conv_maxiter_ = conv_maxiter;
absresid_ = 0;
nupdate_ = 0;
fv_ = 0;
maxiters_ = 0;
combo1_ = 0;
combo2_ = 0;
thislevel_linSys_ = 0; // this level's NOX linear system
nlParams_ = 0; // NOX parameters
initialGuess_ = 0; // NOX initial guess
group_ = 0; // NOX group
solver_ = 0; // NOX solver
SmootherA_ = Mat;
isnlnCG_ = isnlnCG;
azlinSys_ = 0;
clone_ = 0;
nitersCG_ = nitersCG;
broyden_ = broyden;
Broyd_ = 0;
#if 0
if (isnlnCG_==false &&
(fsmoothertype == "Jacobi" ||
smoothertype == "Jacobi" ||
coarsesolvetype == "Jacobi" ))
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: Modified Newton's method not supported for \n"
<< "**ERR**: ismatrixfree_==true && smoothertype == Jacobi-Smoother\n"
<< "**ERR**: because no full Jacobian exists!\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
#endif
if (ismatrixfree_==false)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ismatrixfree_==false on level " << level_ << "\n"
<< "**ERR**: in constructor for ismatrixfree_==true - case\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (!coarseinterface_)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ptr to coarseinterface=NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (!Mat)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ptr to Matrix Mat=NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
Mat->OptimizeStorage();
// ------------------------------------------------------------------------
// get the current solution to this level
xthis_ = coarseinterface_->restrict_fine_to_this(xfine);
// ------------------------------------------------------------------------
// create this level's preconditioner
// We use a 1-level ML-hierarchy for that
ML_Aggregate_Create(&thislevel_ag_);
ML_Create(&thislevel_ml_,1);
// ------------------------------------------------------------------------
// set the Jacobian on level 0 of the local ml
EpetraMatrix2MLMatrix(thislevel_ml_,0,
(dynamic_cast<Epetra_RowMatrix*>(Mat)));
// ------------------------------------------------------------------------
// construct a 1-level ML-hierarchy on this level as a smoother
// ------------------------------------------------------------------------
ML_Set_PrintLevel(ml_printlevel_);
ML_Aggregate_Set_CoarsenScheme_Uncoupled(thislevel_ag_);
ML_Aggregate_Set_DampingFactor(thislevel_ag_, 0.0);
ML_Aggregate_Set_Threshold(thislevel_ag_, 0.0);
ML_Aggregate_Set_MaxCoarseSize(thislevel_ag_,1);
ML_Aggregate_Set_NullSpace(thislevel_ag_,numPDE,nullspdim,NULL,Mat->NumMyRows());
int thislevel_nlevel = ML_Gen_MGHierarchy_UsingAggregation(thislevel_ml_,0,
ML_INCREASING,thislevel_ag_);
if (thislevel_nlevel != 1)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ML generated a local hierarchy of " << thislevel_nlevel << " on level " << level_ << "\n"
<< "**ERR**: this is supposed to be 1 Level only!\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// set the smoother
if (level_==0)
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,fsmoothertype,nsmooth_fine);
else if (level_ != nlevel_-1) // set the smoother from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,smoothertype,nsmooth);
else // set the coarse solver from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,coarsesolvetype,nsmooth_coarse);
// create this level's preconditioner class
ML_Epetra::MultiLevelOperator* ml_tmp = new ML_Epetra::MultiLevelOperator(
thislevel_ml_,comm_,
Mat->OperatorDomainMap(),
Mat->OperatorRangeMap());
thislevel_prec_ = new ML_NOX::ML_Nox_ConstrainedMultiLevelOperator(ml_tmp,*coarseinterface_);
if (!thislevel_prec_)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: thislevel_prec_==NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// intensive test of this level's ML-smoother
#if 0
{
cout << "Test of smoother on level " << level_ << endl;
Epetra_Vector *out = new Epetra_Vector(Copy,*xthis_,0);
out->PutScalar(0.0);
cout << "Input\n";
xthis_->PutScalar(1.0);
Mat->Multiply(false,*xthis_,*out);
xthis_->PutScalar(3.0);
cout << "rhs\n";
cout << *out;
double norm = 0.0;
out->Norm1(&norm);
cout << "Norm of rhs = " << norm << endl;
thislevel_prec_->ApplyInverse(*out,*xthis_);
cout << "result after smoother\n";
cout << *xthis_;
delete out; out = 0;
}
if (level_==2) exit(0);
#endif
// ------------------------------------------------------------------------
// generate this level's coarse prepostoperator
if (level_==0)
coarseprepost_ = new ML_NOX::Ml_Nox_CoarsePrePostOperator(*coarseinterface_,
fineinterface_);
// ------------------------------------------------------------------------
// set up NOX on this level
// ------------------------------------------------------------------------
nlParams_ = new Teuchos::ParameterList();
Teuchos::ParameterList& printParams = nlParams_->sublist("Printing");
printParams.setParameter("MyPID", comm_.MyPID());
printParams.setParameter("Output Precision", 9);
printParams.setParameter("Output Processor", 0);
if (ml_printlevel_>9)
printParams.setParameter("Output Information",
NOX::Utils::OuterIteration +
//NOX::Utils::OuterIterationStatusTest +
//NOX::Utils::InnerIteration +
//NOX::Utils::Parameters +
//NOX::Utils::Details +
NOX::Utils::Warning);
else if (ml_printlevel_>8)
printParams.setParameter("Output Information",
NOX::Utils::Warning);
else
printParams.setParameter("Output Information",0);
if (level_==0)
nlParams_->sublist("Solver Options").setParameter("User Defined Pre/Post Operator", *coarseprepost_);
nlParams_->setParameter("Nonlinear Solver", "Line Search Based");
Teuchos::ParameterList& searchParams = nlParams_->sublist("Line Search");
Teuchos::ParameterList* lsParamsptr = 0;
if (isnlnCG_)
{
searchParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& nlcgParams = dirParams.sublist("Nonlinear CG");
nlcgParams.setParameter("Restart Frequency", 10);
nlcgParams.setParameter("Precondition", "On");
nlcgParams.setParameter("Orthogonalize", "Polak-Ribiere");
//nlcgParams.setParameter("Orthogonalize", "Fletcher-Reeves");
Teuchos::ParameterList& lsParams = nlcgParams.sublist("Linear Solver");
lsParams.setParameter("Aztec Solver", "CG");
lsParams.setParameter("Max Iterations", 1);
lsParams.setParameter("Tolerance", 1e-11);
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
else // Newton's method using ML-preconditioned Aztec as linear solver
{
searchParams.setParameter("Method", "Full Step");
// Sublist for direction
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "Newton");
Teuchos::ParameterList& newtonParams = dirParams.sublist("Newton");
newtonParams.setParameter("Forcing Term Method", "Constant");
//newtonParams.setParameter("Forcing Term Method", "Type 1");
//newtonParams.setParameter("Forcing Term Method", "Type 2");
newtonParams.setParameter("Forcing Term Minimum Tolerance", 1.0e-6);
newtonParams.setParameter("Forcing Term Maximum Tolerance", 0.1);
Teuchos::ParameterList& lsParams = newtonParams.sublist("Linear Solver");
lsParamsptr = &lsParams;
lsParams.setParameter("Aztec Solver", "CG");
lsParams.setParameter("Max Iterations", nitersCG_);
lsParams.setParameter("Tolerance", conv_normF_); // FIXME? is this correct?
if (ml_printlevel_>8)
lsParams.setParameter("Output Frequency", 50);
else
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
// create the initial guess
initialGuess_ = new NOX::Epetra::Vector(*xthis_, NOX::DeepCopy, true);
// NOTE: do not delete xthis_, it's used and destroyed by initialGuess_
// create the necessary interfaces
NOX::EpetraNew::Interface::Preconditioner* iPrec = 0;
NOX::EpetraNew::Interface::Required* iReq = 0;
NOX::EpetraNew::Interface::Jacobian* iJac = 0;
if (isnlnCG_)
{
// create the matrixfree operator used in the nlnCG
thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
// create the necessary interfaces
iPrec = 0;
iReq = coarseinterface_;
iJac = thislevel_A_;
// create the linear system
thislevel_linSys_ = new ML_NOX::Ml_Nox_LinearSystem(
*iJac,*thislevel_A_,*iPrec,
coarseinterface_,*thislevel_prec_,
*xthis_,ismatrixfree_,level_,ml_printlevel_);
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*thislevel_linSys_);
}
else // Modified Newton's method
{
if (!broyden_)
{
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
iJac = this;
// create the initial guess vector
clone_ = new Epetra_Vector(*xthis_);
// create the linear system
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*iJac,*SmootherA_,*iPrec,
*thislevel_prec_,*clone_);
}
else
{
// create the initial guess vector
clone_ = new Epetra_Vector(*xthis_);
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
Broyd_ = new NOX::EpetraNew::BroydenOperator(*nlParams_,*clone_,
*SmootherA_,false);
// create the linear system
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*Broyd_,*SmootherA_,*iPrec,
*thislevel_prec_,*clone_);
}
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*azlinSys_);
}
// create convergence test
create_Nox_Convergencetest(conv_normF_,conv_nupdate_,conv_maxiter_);
// create the solver
solver_ = new NOX::Solver::Manager(*group_,*combo2_,*nlParams_);
return;
}
// ======================================================================
int Amesos_Lapack::GEEV(Epetra_Vector& Er, Epetra_Vector& Ei)
{
if (IsSymbolicFactorizationOK_ == false)
AMESOS_CHK_ERR(SymbolicFactorization());
if (MyPID_ == 0)
AMESOS_CHK_ERR(DenseMatrix_.Shape(static_cast<int>(NumGlobalRows64()),static_cast<int>(NumGlobalRows64())));
AMESOS_CHK_ERR(DistributedToSerial());
AMESOS_CHK_ERR(SerialToDense());
Teuchos::RCP<Epetra_Vector> LocalEr;
Teuchos::RCP<Epetra_Vector> LocalEi;
if (NumProcs_ == 1)
{
LocalEr = Teuchos::rcp(&Er, false);
LocalEi = Teuchos::rcp(&Ei, false);
}
else
{
LocalEr = Teuchos::rcp(new Epetra_Vector(*SerialMap_));
LocalEi = Teuchos::rcp(new Epetra_Vector(*SerialMap_));
}
if (MyPID_ == 0)
{
int n = static_cast<int>(NumGlobalRows64());
char jobvl = 'N'; /* V/N to calculate/not calculate left eigenvectors
of matrix H.*/
char jobvr = 'N'; /* As above, but for right eigenvectors. */
int info = 0;
int ldvl = n;
int ldvr = n;
Er.PutScalar(0.0);
Ei.PutScalar(0.0);
Epetra_LAPACK LAPACK;
std::vector<double> work(1);
int lwork = -1;
LAPACK.GEEV(jobvl, jobvr, n, DenseMatrix_.A(), n,
LocalEr->Values(), LocalEi->Values(), NULL,
ldvl, NULL,
ldvr, &work[0],
lwork, &info);
lwork = (int)work[0];
work.resize(lwork);
LAPACK.GEEV(jobvl, jobvr, n, DenseMatrix_.A(), n,
LocalEr->Values(), LocalEi->Values(), NULL,
ldvl, NULL,
ldvr, &work[0],
lwork, &info);
if (info)
AMESOS_CHK_ERR(info);
}
if (NumProcs_ != 1)
{
// I am not really sure that exporting the results make sense...
// It is just to be coherent with the other parts of the code.
Er.Import(*LocalEr, Epetra_Import(Er.Map(), SerialMap()), Insert);
Ei.Import(*LocalEi, Epetra_Import(Ei.Map(), SerialMap()), Insert);
}
return(0);
}