Teuchos::RCP<Epetra_CrsGraph>
sparse3Tensor2CrsGraph(
const Stokhos::Sparse3Tensor<ordinal_type,value_type>& Cijk,
const Epetra_BlockMap& map)
{
typedef Stokhos::Sparse3Tensor<ordinal_type,value_type> Cijk_type;
// Graph to be created
Teuchos::RCP<Epetra_CrsGraph> graph =
Teuchos::rcp(new Epetra_CrsGraph(Copy, map, 0));
// Loop over Cijk entries including a non-zero in the graph at
// indices (i,j) if there is any k for which Cijk is non-zero
for (typename Cijk_type::k_iterator k_it=Cijk.k_begin();
k_it!=Cijk.k_end(); ++k_it) {
for (typename Cijk_type::kj_iterator j_it = Cijk.j_begin(k_it);
j_it != Cijk.j_end(k_it); ++j_it) {
ordinal_type j = index(j_it);
for (typename Cijk_type::kji_iterator i_it = Cijk.i_begin(j_it);
i_it != Cijk.i_end(j_it); ++i_it) {
ordinal_type i = index(i_it);
graph->InsertGlobalIndices(i, 1, &j);
}
}
}
// Sort, remove redundencies, transform to local, ...
graph->FillComplete();
return graph;
}
/// building the stencil
void PoissonSolver::StencilGeometry(Teuchos::RCP<Epetra_Vector> & RHS,
Teuchos::RCP<Epetra_CrsMatrix> & A)
{
const Epetra_BlockMap & MyMap = RHS->Map();
int NumMyElements = MyMap.NumMyElements();
int* MyGlobalElements = MyMap.MyGlobalElements();
double * rhsvalues = RHS->Values();
std::vector<double> Values(5);
std::vector<int> Indices(5);
const auto & inside = _bend.getInsideMask();
for (int lid = 0; lid < NumMyElements; ++ lid) {
size_t NumEntries = 0;
const size_t & gid = MyGlobalElements[lid];
cutoffStencil(Indices,
Values,
rhsvalues[lid],
NumEntries,
inside,
gid);
A->InsertGlobalValues(gid, NumEntries, &Values[0], &Indices[0]);
}
A->FillComplete();
A->OptimizeStorage();
}
Teuchos::RCP<Epetra_CrsMatrix> buildMatrix(int nx, Epetra_Comm & comm)
{
Epetra_Map map(nx*comm.NumProc(),0,comm);
Teuchos::RCP<Epetra_CrsMatrix> mat = Teuchos::rcp(new Epetra_CrsMatrix(Copy,map,3));
int offsets[3] = {-1, 0, 1 };
double values[3] = { -1, 2, -1};
int maxGid = map.MaxAllGID();
for(int lid=0;lid<nx;lid++) {
int gid = mat->GRID(lid);
int numEntries = 3, offset = 0;
int indices[3] = { gid+offsets[0],
gid+offsets[1],
gid+offsets[2] };
if(gid==0) { // left end point
numEntries = 2;
offset = 1;
} // right end point
else if(gid==maxGid)
numEntries = 2;
// insert rows
mat->InsertGlobalValues(gid,numEntries,values+offset,indices+offset);
}
mat->FillComplete();
return mat;
}
//---------------------------------------------------------------------------//
TEUCHOS_UNIT_TEST( EpetraPointJacobiPreconditioner, tridiag_matrix )
{
typedef Epetra_RowMatrix MatrixType;
typedef Epetra_Vector VectorType;
typedef MCLS::VectorTraits<VectorType> VT;
typedef MCLS::MatrixTraits<VectorType,MatrixType> MT;
Teuchos::RCP<const Teuchos::Comm<int> > comm =
Teuchos::DefaultComm<int>::getComm();
Teuchos::RCP<Epetra_Comm> epetra_comm = getEpetraComm( comm );
int comm_size = comm->getSize();
int local_num_rows = 10;
int global_num_rows = local_num_rows*comm_size;
Teuchos::RCP<Epetra_Map> map = Teuchos::rcp(
new Epetra_Map( global_num_rows, 0, *epetra_comm ) );
Teuchos::RCP<Epetra_CrsMatrix> A =
Teuchos::rcp( new Epetra_CrsMatrix( Copy, *map, 0 ) );
Teuchos::Array<int> global_columns( 3 );
double diag_val = 2.0;
Teuchos::Array<double> values( 3, diag_val );
for ( int i = 1; i < global_num_rows-1; ++i )
{
global_columns[0] = i-1;
global_columns[1] = i;
global_columns[2] = i+1;
A->InsertGlobalValues( i, global_columns().size(),
&values[0], &global_columns[0] );
}
Teuchos::Array<int> single_col(1,0);
Teuchos::Array<double> diag_elem(1,diag_val);
A->InsertGlobalValues( 0, 1, diag_elem.getRawPtr(), single_col.getRawPtr() );
single_col[0] = global_num_rows-1;
A->InsertGlobalValues( global_num_rows-1, 1, diag_elem.getRawPtr(),
single_col.getRawPtr() );
A->FillComplete();
// Build the preconditioner.
Teuchos::RCP<MCLS::Preconditioner<MatrixType> > preconditioner =
Teuchos::rcp( new MCLS::EpetraPointJacobiPreconditioner() );
preconditioner->setOperator( A );
preconditioner->buildPreconditioner();
Teuchos::RCP<const MatrixType> M = preconditioner->getLeftPreconditioner();
// Check the preconditioner.
Teuchos::RCP<VectorType> X = MT::cloneVectorFromMatrixRows(*A);
MT::getLocalDiagCopy( *M, *X );
Teuchos::ArrayRCP<const double> X_view = VT::view( *X );
Teuchos::ArrayRCP<const double>::const_iterator view_iterator;
for ( view_iterator = X_view.begin();
view_iterator != X_view.end();
++view_iterator )
{
TEST_EQUALITY( *view_iterator, 1.0/diag_val );
}
}
Teuchos::RCP<Epetra_Operator>
Albany::SolutionResponseFunction::
createGradientOp() const
{
Teuchos::RCP<Epetra_CrsMatrix> G =
Teuchos::rcp(new Epetra_CrsMatrix(Copy, *gradient_graph));
G->FillComplete();
return G;
}
Teuchos::RCP<Epetra_Operator>
twoD_diffusion_ME::
create_W() const
{
Teuchos::RCP<Epetra_CrsMatrix> AA =
Teuchos::rcp(new Epetra_CrsMatrix(Copy, *graph));
AA->FillComplete();
AA->OptimizeStorage();
return AA;
}
Teuchos::RCP<Epetra_CrsMatrix>
buildH( const Teuchos::RCP<Epetra_CrsMatrix> &A)
{
Teuchos::RCP<Epetra_CrsMatrix> H = Teuchos::rcp(
new Epetra_CrsMatrix(Copy, A->RowMap(), A->GlobalMaxNumEntries() ) );
int N = A->NumGlobalRows();
std::vector<double> A_values( N );
std::vector<int> A_indices( N );
int A_size = 0;
double local_H;
bool found_diag = false;
for ( int i = 0; i < N; ++i )
{
A->ExtractGlobalRowCopy( i,
N,
A_size,
&A_values[0],
&A_indices[0] );
for ( int j = 0; j < A_size; ++j )
{
if ( i == A_indices[j] )
{
local_H = 1.0 - A_values[j];
H->InsertGlobalValues( i, 1, &local_H, &A_indices[j] );
found_diag = true;
}
else
{
local_H = -A_values[j];
H->InsertGlobalValues( i, 1, &local_H, &A_indices[j] );
}
}
if ( !found_diag )
{
local_H = 1.0;
H->InsertGlobalValues( i, 1, &local_H, &i );
}
}
H->FillComplete();
return H;
}
Teuchos::RCP<Epetra_CrsGraph>
sparse3Tensor2CrsGraph(
const Stokhos::OrthogPolyBasis<ordinal_type,value_type>& basis,
const Stokhos::Sparse3Tensor<ordinal_type,value_type>& Cijk,
const Epetra_Comm& comm)
{
// Number of stochastic rows
ordinal_type num_rows = basis.size();
// Replicated local map
Epetra_LocalMap map(num_rows, 0, comm);
// Graph to be created
Teuchos::RCP<Epetra_CrsGraph> graph =
Teuchos::rcp(new Epetra_CrsGraph(Copy, map, 0));
// Loop over Cijk entries including a non-zero in the graph at
// indices (i,j) if there is any k for which Cijk is non-zero
ordinal_type Cijk_size = Cijk.size();
for (ordinal_type k=0; k<Cijk_size; k++) {
ordinal_type nj = Cijk.num_j(k);
const Teuchos::Array<int>& j_indices = Cijk.Jindices(k);
for (ordinal_type jj=0; jj<nj; jj++) {
ordinal_type j = j_indices[jj];
const Teuchos::Array<int>& i_indices = Cijk.Iindices(k,jj);
ordinal_type ni = i_indices.size();
for (ordinal_type ii=0; ii<ni; ii++) {
ordinal_type i = i_indices[ii];
graph->InsertGlobalIndices(i, 1, &j);
}
}
}
// Sort, remove redundencies, transform to local, ...
graph->FillComplete();
return graph;
}
Teuchos::RCP<Epetra_CrsMatrix> Epetra_Operator_to_Epetra_Matrix::constructInverseMatrix(const Epetra_Operator &op, const Epetra_Map &map) {
int numEntriesPerRow = 0;
Teuchos::RCP<Epetra_CrsMatrix> matrix = Teuchos::rcp(new Epetra_CrsMatrix(::Copy, map, numEntriesPerRow));
int numRows = map.NumGlobalElements();
Epetra_Vector X(map);
Epetra_Vector Y(map);
double tol = 1e-15; // values below this will be considered 0
for (int rowIndex=0; rowIndex<numRows; rowIndex++) {
int lid = map.LID(rowIndex);
if (lid != -1) {
X[lid] = 1.0;
}
op.ApplyInverse(X, Y);
if (lid != -1) {
X[lid] = 0.0;
}
std::vector<double> values;
std::vector<int> indices;
for (int i=0; i<map.NumMyElements(); i++) {
if (abs(Y[i]) > tol) {
values.push_back(Y[i]);
indices.push_back(map.GID(i));
}
}
matrix->InsertGlobalValues(rowIndex, values.size(), &values[0], &indices[0]);
}
matrix->FillComplete();
return matrix;
}
TEUCHOS_UNIT_TEST( OperatorTools, spectral_radius_test)
{
int problem_size = 100;
Epetra_SerialComm comm;
Epetra_Map map( problem_size, 0, comm );
// Build A.
Teuchos::RCP<Epetra_CrsMatrix> A =
Teuchos::rcp( new Epetra_CrsMatrix( Copy, map, problem_size ) );
double lower_diag = -1.0;
double diag = 2.0;
double upper_diag = -1.0;
int global_row = 0;
int lower_row = 0;
int upper_row = 0;
for ( int i = 0; i < problem_size; ++i )
{
global_row = A->GRID(i);
lower_row = i-1;
upper_row = i+1;
if ( lower_row > -1 )
{
A->InsertGlobalValues( global_row, 1, &lower_diag, &lower_row );
}
A->InsertGlobalValues( global_row, 1, &diag, &global_row );
if ( upper_row < problem_size )
{
A->InsertGlobalValues( global_row, 1, &upper_diag, &upper_row );
}
}
A->FillComplete();
double spec_rad_A = HMCSA::OperatorTools::spectralRadius( A );
std::cout << spec_rad_A << std::endl;
}
int main(int argc, char *argv[])
{
int i;
bool ierr, gerr;
gerr = true;
#ifdef HAVE_MPI
// Initialize MPI and setup an Epetra communicator
MPI_Init(&argc,&argv);
Teuchos::RCP<Epetra_MpiComm> Comm = Teuchos::rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
// If we aren't using MPI, then setup a serial communicator.
Teuchos::RCP<Epetra_SerialComm> Comm = Teuchos::rcp( new Epetra_SerialComm() );
#endif
// number of global elements
int dim = 100;
int blockSize = 5;
bool verbose = false;
if (argc>1) {
if (argv[1][0]=='-' && argv[1][1]=='v') {
verbose = true;
}
}
// Construct a Map that puts approximately the same number of
// equations on each processor.
Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( new Epetra_Map(dim, 0, *Comm) );
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map->NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map->MyGlobalElements(&MyGlobalElements[0]);
// Create an integer std::vector NumNz that is used to build the Petra Matrix.
// NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation
// on this processor
std::vector<int> NumNz(NumMyElements);
// We are building a tridiagonal matrix where each row has (-1 2 -1)
// So we need 2 off-diagonal terms (except for the first and last equation)
for (i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i]==0 || MyGlobalElements[i] == dim-1) {
NumNz[i] = 2;
}
else {
NumNz[i] = 3;
}
}
// Create an Epetra_Matrix
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix(Copy, *Map, &NumNz[0]) );
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
double two = 2.0;
int NumEntries;
for (i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == dim-1) {
Indices[0] = dim-2;
NumEntries = 1;
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
ierr = A->InsertGlobalValues(MyGlobalElements[i],NumEntries,&Values[0],&Indices[0]);
assert(ierr==0);
// Put in the diagonal entry
ierr = A->InsertGlobalValues(MyGlobalElements[i],1,&two,&MyGlobalElements[i]);
assert(ierr==0);
}
// Finish building the epetra matrix A
ierr = A->FillComplete();
assert(ierr==0);
// Issue several useful typedefs;
typedef Belos::MultiVec<double> EMV;
typedef Belos::Operator<double> EOP;
// Create an Epetra_MultiVector for an initial std::vector to start the solver.
// Note that this needs to have the same number of columns as the blocksize.
Teuchos::RCP<Belos::EpetraMultiVec> ivec = Teuchos::rcp( new Belos::EpetraMultiVec(*Map, blockSize) );
ivec->Random();
// Create an output manager to handle the I/O from the solver
Teuchos::RCP<Belos::OutputManager<double> > MyOM = Teuchos::rcp( new Belos::OutputManager<double>() );
if (verbose) {
MyOM->setVerbosity( Belos::Warnings );
}
// test the Epetra adapter multivector
ierr = Belos::TestMultiVecTraits<double,EMV>(MyOM,ivec);
gerr &= ierr;
if (ierr) {
MyOM->print(Belos::Warnings,"*** EpetraAdapter PASSED TestMultiVecTraits()\n");
}
else {
MyOM->print(Belos::Warnings,"*** EpetraAdapter FAILED TestMultiVecTraits() ***\n\n");
}
#ifdef HAVE_MPI
MPI_Finalize();
#endif
if (gerr == false) {
MyOM->print(Belos::Warnings,"End Result: TEST FAILED\n");
return -1;
}
//
// Default return value
//
MyOM->print(Belos::Warnings,"End Result: TEST PASSED\n");
return 0;
}
//---------------------------------------------------------------------------//
// Test templates
//---------------------------------------------------------------------------//
TEUCHOS_UNIT_TEST( SolverFactory, mcsa_two_by_two )
{
typedef Epetra_Vector VectorType;
typedef MCLS::VectorTraits<VectorType> VT;
typedef Epetra_RowMatrix MatrixType;
typedef MCLS::MatrixTraits<VectorType,MatrixType> MT;
Teuchos::RCP<const Teuchos::Comm<int> > comm =
Teuchos::DefaultComm<int>::getComm();
int comm_size = comm->getSize();
int comm_rank = comm->getRank();
// This is a 4 processor test.
if ( comm_size == 4 )
{
// Build the set-constant communicator.
Teuchos::Array<int> ranks(2);
if ( comm_rank < 2 )
{
ranks[0] = 0;
ranks[1] = 1;
}
else
{
ranks[0] = 2;
ranks[1] = 3;
}
Teuchos::RCP<const Teuchos::Comm<int> > comm_set =
comm->createSubcommunicator( ranks() );
int set_size = comm_set->getSize();
// Declare the linear problem in the global scope.
Teuchos::RCP<MCLS::LinearProblem<VectorType,MatrixType> > linear_problem;
// Build the linear system on set 0.
if ( comm_rank < 2 )
{
int local_num_rows = 10;
int global_num_rows = local_num_rows*set_size;
Teuchos::RCP<Epetra_Comm> epetra_comm = getEpetraComm( comm_set );
Teuchos::RCP<Epetra_Map> map = Teuchos::rcp(
new Epetra_Map( global_num_rows, 0, *epetra_comm ) );
// Build the linear system. This operator is symmetric with a spectral
// radius less than 1.
Teuchos::RCP<Epetra_CrsMatrix> A =
Teuchos::rcp( new Epetra_CrsMatrix( Copy, *map, 0 ) );
Teuchos::Array<int> global_columns( 3 );
Teuchos::Array<double> values( 3 );
global_columns[0] = 0;
global_columns[1] = 1;
global_columns[2] = 2;
values[0] = 1.0;
values[1] = 0.05;
values[2] = 0.05;
A->InsertGlobalValues( 0, global_columns.size(),
&values[0], &global_columns[0] );
for ( int i = 1; i < global_num_rows-1; ++i )
{
global_columns[0] = i-1;
global_columns[1] = i;
global_columns[2] = i+1;
values[0] = 0.05;
values[1] = 1.0;
values[2] = 0.05;
A->InsertGlobalValues( i, global_columns.size(),
&values[0], &global_columns[0] );
}
global_columns[0] = global_num_rows-3;
global_columns[1] = global_num_rows-2;
global_columns[2] = global_num_rows-1;
values[0] = 0.05;
values[1] = 0.05;
values[2] = 1.0;
A->InsertGlobalValues( global_num_rows-1, global_columns.size(),
&values[0], &global_columns[0] );
A->FillComplete();
Teuchos::RCP<MatrixType> B = A;
// Build the LHS. Put a large positive number here to be sure we are
// clear the vector before solving.
Teuchos::RCP<VectorType> x = MT::cloneVectorFromMatrixRows( *B );
VT::putScalar( *x, 0.0 );
// Build the RHS with negative numbers. this gives us a negative
// solution.
Teuchos::RCP<VectorType> b = MT::cloneVectorFromMatrixRows( *B );
VT::putScalar( *b, -1.0 );
// Create the linear problem.
linear_problem = Teuchos::rcp(
new MCLS::LinearProblem<VectorType,MatrixType>(B, x, b) );
}
comm->barrier();
// Solver parameters.
Teuchos::RCP<Teuchos::ParameterList> plist =
Teuchos::rcp( new Teuchos::ParameterList() );
double cutoff = 1.0e-4;
plist->set<std::string>("MC Type", "Adjoint");
plist->set<double>("Convergence Tolerance", 1.0e-8);
plist->set<int>("Maximum Iterations", 10);
plist->set<double>("Weight Cutoff", cutoff);
plist->set<int>("Iteration Print Frequency", 1);
plist->set<int>("MC Check Frequency", 50);
plist->set<bool>("Reproducible MC Mode",true);
plist->set<int>("Overlap Size", 2);
plist->set<int>("Number of Sets", 2);
plist->set<double>("Sample Ratio", 10.0);
plist->set<std::string>("Transport Type", "Global" );
// Create the solver.
MCLS::SolverFactory<VectorType,MatrixType> factory;
Teuchos::RCP<MCLS::SolverManager<VectorType,MatrixType> > solver_manager =
factory.create( "MCSA", comm, plist );
solver_manager->setProblem( linear_problem );
// Solve the problem.
bool converged_status = solver_manager->solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager->getConvergedStatus() );
TEST_ASSERT( solver_manager->getNumIters() < 10 );
if ( comm_rank < 2 )
{
TEST_ASSERT( solver_manager->achievedTol() > 0.0 );
}
else
{
TEST_ASSERT( solver_manager->achievedTol() == 0.0 );
}
if ( comm_rank < 2 )
{
// Check that we got a negative solution.
Teuchos::ArrayRCP<const double> x_view =
VT::view( *linear_problem->getLHS() );
Teuchos::ArrayRCP<const double>::const_iterator x_view_it;
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it < Teuchos::ScalarTraits<double>::zero() );
}
}
comm->barrier();
// Now solve the problem with a positive source.
if ( comm_rank < 2 )
{
Teuchos::RCP<VectorType> b =
MT::cloneVectorFromMatrixRows( *linear_problem->getOperator() );
VT::putScalar( *b, 2.0 );
linear_problem->setRHS( b );
VT::putScalar( *linear_problem->getLHS(), 0.0 );
}
comm->barrier();
converged_status = solver_manager->solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager->getConvergedStatus() );
TEST_ASSERT( solver_manager->getNumIters() < 10 );
if ( comm_rank < 2 )
{
TEST_ASSERT( solver_manager->achievedTol() > 0.0 );
}
else
{
TEST_ASSERT( solver_manager->achievedTol() == 0.0 );
}
if ( comm_rank < 2 )
{
Teuchos::ArrayRCP<const double> x_view =
VT::view( *linear_problem->getLHS() );
Teuchos::ArrayRCP<const double>::const_iterator x_view_it;
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it > Teuchos::ScalarTraits<double>::zero() );
}
}
comm->barrier();
// Reset the domain and solve again with a positive source.
if ( comm_rank < 2 )
{
VT::putScalar( *linear_problem->getLHS(), 0.0 );
}
comm->barrier();
solver_manager->setProblem( linear_problem );
converged_status = solver_manager->solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager->getConvergedStatus() );
TEST_ASSERT( solver_manager->getNumIters() < 10 );
if ( comm_rank < 2 )
{
TEST_ASSERT( solver_manager->achievedTol() > 0.0 );
}
else
{
TEST_ASSERT( solver_manager->achievedTol() == 0.0 );
}
if ( comm_rank < 2 )
{
Teuchos::ArrayRCP<const double> x_view =
VT::view( *linear_problem->getLHS() );
Teuchos::ArrayRCP<const double>::const_iterator x_view_it;
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it > Teuchos::ScalarTraits<double>::zero() );
}
}
comm->barrier();
// Reset both and solve with a negative source.
if ( comm_rank < 2 )
{
Teuchos::RCP<VectorType> b =
MT::cloneVectorFromMatrixRows( *linear_problem->getOperator() );
VT::putScalar( *b, -2.0 );
linear_problem->setRHS( b );
VT::putScalar( *linear_problem->getLHS(), 0.0 );
}
comm->barrier();
converged_status = solver_manager->solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager->getConvergedStatus() );
TEST_ASSERT( solver_manager->getNumIters() < 10 );
if ( comm_rank < 2 )
{
TEST_ASSERT( solver_manager->achievedTol() > 0.0 );
}
else
{
TEST_ASSERT( solver_manager->achievedTol() == 0.0 );
}
if ( comm_rank < 2 )
{
Teuchos::ArrayRCP<const double> x_view =
VT::view( *linear_problem->getLHS() );
Teuchos::ArrayRCP<const double>::const_iterator x_view_it;
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it < Teuchos::ScalarTraits<double>::zero() );
}
}
comm->barrier();
}
}
//=============================================================================
int Amesos_Dscpack::PerformSymbolicFactorization()
{
ResetTimer(0);
ResetTimer(1);
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ;
const Epetra_MpiComm& comm1 = dynamic_cast<const Epetra_MpiComm &> (Comm());
int numrows = RowMatrixA->NumGlobalRows();
int numentries = RowMatrixA->NumGlobalNonzeros();
Teuchos::RCP<Epetra_CrsGraph> Graph;
Epetra_CrsMatrix* CastCrsMatrixA =
dynamic_cast<Epetra_CrsMatrix*>(RowMatrixA);
if (CastCrsMatrixA)
{
Graph = Teuchos::rcp(const_cast<Epetra_CrsGraph*>(&(CastCrsMatrixA->Graph())), false);
}
else
{
int MaxNumEntries = RowMatrixA->MaxNumEntries();
Graph = Teuchos::rcp(new Epetra_CrsGraph(Copy, OriginalMap, MaxNumEntries));
std::vector<int> Indices(MaxNumEntries);
std::vector<double> Values(MaxNumEntries);
for (int i = 0 ; i < RowMatrixA->NumMyRows() ; ++i)
{
int NumEntries;
RowMatrixA->ExtractMyRowCopy(i, MaxNumEntries, NumEntries,
&Values[0], &Indices[0]);
for (int j = 0 ; j < NumEntries ; ++j)
Indices[j] = RowMatrixA->RowMatrixColMap().GID(Indices[j]);
int GlobalRow = RowMatrixA->RowMatrixRowMap().GID(i);
Graph->InsertGlobalIndices(GlobalRow, NumEntries, &Indices[0]);
}
Graph->FillComplete();
}
//
// Create a replicated map and graph
//
std::vector<int> AllIDs( numrows ) ;
for ( int i = 0; i < numrows ; i++ ) AllIDs[i] = i ;
Epetra_Map ReplicatedMap( -1, numrows, &AllIDs[0], 0, Comm());
Epetra_Import ReplicatedImporter(ReplicatedMap, OriginalMap);
Epetra_CrsGraph ReplicatedGraph( Copy, ReplicatedMap, 0 );
AMESOS_CHK_ERR(ReplicatedGraph.Import(*Graph, ReplicatedImporter, Insert));
AMESOS_CHK_ERR(ReplicatedGraph.FillComplete());
//
// Convert the matrix to Ap, Ai
//
std::vector <int> Replicates(numrows);
std::vector <int> Ap(numrows + 1);
std::vector <int> Ai(EPETRA_MAX(numrows, numentries));
for( int i = 0 ; i < numrows; i++ ) Replicates[i] = 1;
int NumEntriesPerRow ;
int *ColIndices = 0 ;
int Ai_index = 0 ;
for ( int MyRow = 0; MyRow <numrows; MyRow++ ) {
AMESOS_CHK_ERR( ReplicatedGraph.ExtractMyRowView( MyRow, NumEntriesPerRow, ColIndices ) );
Ap[MyRow] = Ai_index ;
for ( int j = 0; j < NumEntriesPerRow; j++ ) {
Ai[Ai_index] = ColIndices[j] ;
Ai_index++;
}
}
assert( Ai_index == numentries ) ;
Ap[ numrows ] = Ai_index ;
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0);
ResetTimer(0);
//
// Call Dscpack Symbolic Factorization
//
int OrderCode = 2;
std::vector<double> MyANonZ;
NumLocalNonz = 0 ;
GlobalStructNewColNum = 0 ;
GlobalStructNewNum = 0 ;
GlobalStructOwner = 0 ;
LocalStructOldNum = 0 ;
NumGlobalCols = 0 ;
// MS // Have to define the maximum number of processes to be used
// MS // This is only a suggestion as Dscpack uses a number of processes that is a power of 2
int NumGlobalNonzeros = GetProblem()->GetMatrix()->NumGlobalNonzeros();
int NumRows = GetProblem()->GetMatrix()->NumGlobalRows();
// optimal value for MaxProcs == -1
int OptNumProcs1 = 1+EPETRA_MAX( NumRows/10000, NumGlobalNonzeros/1000000 );
OptNumProcs1 = EPETRA_MIN(NumProcs_,OptNumProcs1 );
// optimal value for MaxProcs == -2
int OptNumProcs2 = (int)sqrt(1.0 * NumProcs_);
if( OptNumProcs2 < 1 ) OptNumProcs2 = 1;
// fix the value of MaxProcs
switch (MaxProcs_)
{
case -1:
MaxProcs_ = EPETRA_MIN(OptNumProcs1, NumProcs_);
break;
case -2:
MaxProcs_ = EPETRA_MIN(OptNumProcs2, NumProcs_);
break;
case -3:
MaxProcs_ = NumProcs_;
break;
}
#if 0
if (MyDscRank>=0 && A_and_LU_built) {
DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject);
}
#endif
// if ( ! A_and_LU_built ) {
// DSC_End( PrivateDscpackData_->MyDSCObject ) ;
// PrivateDscpackData_->MyDSCObject = DSC_Begin() ;
// }
// MS // here I continue with the old code...
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
DscNumProcs = 1 ;
int DscMax = DSC_Analyze( numrows, &Ap[0], &Ai[0], &Replicates[0] );
while ( DscNumProcs * 2 <=EPETRA_MIN( MaxProcs_, DscMax ) ) DscNumProcs *= 2 ;
MyDscRank = -1;
DSC_Open0( PrivateDscpackData_->MyDSCObject_, DscNumProcs, &MyDscRank, comm1.Comm()) ;
NumLocalCols = 0 ; // This is for those processes not in the Dsc grid
if ( MyDscRank >= 0 ) {
assert( MyPID_ == MyDscRank ) ;
AMESOS_CHK_ERR( DSC_Order ( PrivateDscpackData_->MyDSCObject_, OrderCode, numrows, &Ap[0], &Ai[0],
&Replicates[0], &NumGlobalCols, &NumLocalStructs,
&NumLocalCols, &NumLocalNonz,
&GlobalStructNewColNum, &GlobalStructNewNum,
&GlobalStructOwner, &LocalStructOldNum ) ) ;
assert( NumGlobalCols == numrows ) ;
assert( NumLocalCols == NumLocalStructs ) ;
}
if ( MyDscRank >= 0 ) {
int MaxSingleBlock;
const int Limit = 5000000 ; // Memory Limit set to 5 Terabytes
AMESOS_CHK_ERR( DSC_SFactor ( PrivateDscpackData_->MyDSCObject_, &TotalMemory_,
&MaxSingleBlock, Limit, DSC_LBLAS3, DSC_DBLAS2 ) ) ;
}
// A_and_LU_built = true; // If you uncomment this, TestOptions fails
SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_, 0);
return(0);
}
/* Apply an identity matrix to the Schur complement operator. Drop the entries
entries using a relative threshold. Assemble the result in a Crs Matrix
which will be our approximate Schur complement.
*/
Teuchos::RCP<Epetra_CrsMatrix> computeApproxSchur(shylu_config *config,
shylu_symbolic *sym,
Epetra_CrsMatrix *G, Epetra_CrsMatrix *R,
Epetra_LinearProblem *LP, Amesos_BaseSolver *solver,
Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C,
Epetra_Map *localDRowMap)
{
double relative_thres = config->relative_threshold;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, sym, G, R, LP, solver, ifSolver, C,
localDRowMap, nvectors);
// Get row map
Epetra_Map rMap = G->RowMap();
int *rows = rMap.MyGlobalElements();
int totalElems = rMap.NumGlobalElements();
int localElems = rMap.NumMyElements();
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
int prefixSum;
G->Comm().ScanSum(&localElems, &prefixSum, 1);
//cout << " prefixSum" << prefixSum << endl;
// Start the index in prefixSum-localElems
int *mySGID = new int[totalElems]; // vector of size Schur complement !
int *allSGID = new int[totalElems]; // vector of size Schur complement !
int i, j;
for (i = 0, j = 0; i < totalElems ; i++)
{
if (i >= prefixSum - localElems && i < prefixSum)
{
mySGID[i] = rows[j];
j++;
}
else
{
mySGID[i] = 0;
}
allSGID[i] = 0;
}
C->Comm().SumAll(mySGID, allSGID, totalElems);
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time to Compute RowIDS" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
// Now everyone knows the GIDs in the Schur complement
//cout << rMap << endl;
j = 0;
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, rMap, localElems));
int nentries;
double *values = new double[localElems]; // Need to adjust this for more
int *indices = new int[localElems]; // than one vector
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("Apply time");
#endif
int findex = totalElems / nvectors ;
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
Epetra_MultiVector probevec(rMap, nvectors);
Epetra_MultiVector Scol(rMap, nvectors);
probevec.PutScalar(0.0);
int cindex;
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
if (cindex >= prefixSum - localElems && cindex < prefixSum)
{
probevec.ReplaceGlobalValue(allSGID[cindex], k, 1.0);
}
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (j = 0 ; j < localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (allSGID[cindex] == rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[cindex];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[cindex];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < totalElems ; i++)
{
Epetra_MultiVector probevec(rMap, 1); // TODO: Try doing more than one
Epetra_MultiVector Scol(rMap, 1); // vector at a time
probevec.PutScalar(0.0);
if (i >= prefixSum - localElems && i < prefixSum)
{
probevec.ReplaceGlobalValue(allSGID[i], 0, 1.0);
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (j = 0 ; j < localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (allSGID[i] == rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[i];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[i];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
Sbar->FillComplete();
cout << "#dropped entries" << dropped << endl;
delete[] allSGID;
delete[] mySGID;
delete[] values;
delete[] indices;
delete[] maxvalue;
return Sbar;
}
//---------------------------------------------------------------------------//
// Test templates
//---------------------------------------------------------------------------//
TEUCHOS_UNIT_TEST( FixedPointSolverManager, one_by_one )
{
typedef Epetra_Vector VectorType;
typedef MCLS::VectorTraits<VectorType> VT;
typedef Epetra_RowMatrix MatrixType;
typedef MCLS::MatrixTraits<VectorType,MatrixType> MT;
Teuchos::RCP<const Teuchos::Comm<int> > comm =
Teuchos::DefaultComm<int>::getComm();
Teuchos::RCP<Epetra_Comm> epetra_comm = getEpetraComm( comm );
int comm_size = comm->getSize();
int local_num_rows = 10;
int global_num_rows = local_num_rows*comm_size;
Teuchos::RCP<Epetra_Map> map = Teuchos::rcp(
new Epetra_Map( global_num_rows, 0, *epetra_comm ) );
// Build the linear system.
Teuchos::RCP<Epetra_CrsMatrix> A =
Teuchos::rcp( new Epetra_CrsMatrix( Copy, *map, 0 ) );
Teuchos::Array<int> global_columns( 3 );
Teuchos::Array<double> values( 3 );
global_columns[0] = 0;
global_columns[1] = 1;
global_columns[2] = 2;
values[0] = 0.14;
values[1] = 0.14;
values[2] = 1.0;
A->InsertGlobalValues( 0, global_columns.size(),
&values[0], &global_columns[0] );
for ( int i = 1; i < global_num_rows-1; ++i )
{
global_columns[0] = i-1;
global_columns[1] = i;
global_columns[2] = i+1;
values[0] = 0.14;
values[1] = 1.0;
values[2] = 0.14;
A->InsertGlobalValues( i, global_columns.size(),
&values[0], &global_columns[0] );
}
global_columns[0] = global_num_rows-3;
global_columns[1] = global_num_rows-2;
global_columns[2] = global_num_rows-1;
values[0] = 0.14;
values[1] = 0.14;
values[2] = 1.0;
A->InsertGlobalValues( global_num_rows-1, global_columns.size(),
&values[0], &global_columns[0] );
A->FillComplete();
Teuchos::RCP<MatrixType> B = A;
// Build the LHS. Put a large positive number here to be sure we are
// clear the vector before solving.
Teuchos::RCP<VectorType> x = MT::cloneVectorFromMatrixRows( *B );
VT::putScalar( *x, 0.0 );
// Build the RHS with negative numbers. this gives us a negative
// solution.
Teuchos::RCP<VectorType> b = MT::cloneVectorFromMatrixRows( *B );
VT::putScalar( *b, -1.0 );
// Solver parameters.
Teuchos::RCP<Teuchos::ParameterList> plist =
Teuchos::rcp( new Teuchos::ParameterList() );
plist->set<int>("Iteration Print Frequency", 1);
plist->set<double>("Convergence Tolerance", 1.0e-8);
plist->set<int>("Maximum Iterations", 100);
plist->set<double>("Richardson Relaxation", 1.0);
// Create the linear problem.
Teuchos::RCP<MCLS::LinearProblem<VectorType,MatrixType> > linear_problem =
Teuchos::rcp( new MCLS::LinearProblem<VectorType,MatrixType>(
B, x, b ) );
// Create the solver.
MCLS::FixedPointSolverManager<VectorType,MatrixType>
solver_manager( linear_problem, comm, plist );
// Solve the problem.
bool converged_status = solver_manager.solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager.getConvergedStatus() );
TEST_EQUALITY( solver_manager.getNumIters(), 15 );
TEST_ASSERT( solver_manager.achievedTol() > 0.0 );
// Check that we got a negative solution.
Teuchos::ArrayRCP<const double> x_view = VT::view(*x);
Teuchos::ArrayRCP<const double>::const_iterator x_view_it;
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it < Teuchos::ScalarTraits<double>::zero() );
}
// Now solve the problem with a positive source.
VT::putScalar( *b, 2.0 );
VT::putScalar( *x, 0.0 );
linear_problem->setLHS(x);
converged_status = solver_manager.solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager.getConvergedStatus() );
TEST_EQUALITY( solver_manager.getNumIters(), 15 );
TEST_ASSERT( solver_manager.achievedTol() > 0.0 );
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it > Teuchos::ScalarTraits<double>::zero() );
}
// Reset the domain and solve again with a positive source.
VT::putScalar( *x, 0.0 );
solver_manager.setProblem( linear_problem );
linear_problem->setLHS(x);
converged_status = solver_manager.solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager.getConvergedStatus() );
TEST_EQUALITY( solver_manager.getNumIters(), 15 );
TEST_ASSERT( solver_manager.achievedTol() > 0.0 );
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it > Teuchos::ScalarTraits<double>::zero() );
}
// Reset both and solve with a negative source.
VT::putScalar( *b, -2.0 );
VT::putScalar( *x, 0.0 );
linear_problem->setLHS(x);
converged_status = solver_manager.solve();
TEST_ASSERT( converged_status );
TEST_ASSERT( solver_manager.getConvergedStatus() );
TEST_EQUALITY( solver_manager.getNumIters(), 15 );
TEST_ASSERT( solver_manager.achievedTol() > 0.0 );
for ( x_view_it = x_view.begin(); x_view_it != x_view.end(); ++x_view_it )
{
TEST_ASSERT( *x_view_it < Teuchos::ScalarTraits<double>::zero() );
}
}
int extract_matrices
(
Epetra_CrsMatrix *A, // i/p: A matrix
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure, TODO: Required ?
shylu_config *config, // i/p: library configuration
bool insertValues // true implies values will be inserted and fill
// complete will be called. false implies values
// will be replaced.
)
{
Teuchos::RCP<Epetra_CrsMatrix> D = ssym->D;
Teuchos::RCP<Epetra_CrsMatrix> C = ssym->C;
Teuchos::RCP<Epetra_CrsMatrix> R = ssym->R;
Teuchos::RCP<Epetra_CrsMatrix> G = ssym->G;
Teuchos::RCP<Epetra_CrsGraph> Sg = ssym->Sg;
int *DColElems = data->DColElems;
int *gvals = data->gvals;
double Sdiagfactor = config->Sdiagfactor;
int *LeftIndex = new int[data->lmax];
double *LeftValues = new double[data->lmax];
int *RightIndex = new int[data->rmax];
double *RightValues = new double[data->rmax];
int err;
int lcnt, rcnt ;
int gcid;
int gid;
int *Ai;
double *Ax;
int nrows = A->RowMap().NumMyElements();
int *rows = A->RowMap().MyGlobalElements();
for (int i = 0; i < nrows ; i++)
{
int NumEntries;
err = A->ExtractMyRowView(i, NumEntries, Ax, Ai);
lcnt = 0; rcnt = 0;
// Place the entry in the correct sub matrix, Works only for sym
gid = rows[i];
int lcid;
for (int j = 0 ; j < NumEntries ; j++)
{ // O(nnz) ! Careful what you do inside
// Row permutation does not matter here
gcid = A->GCID(Ai[j]);
assert(gcid != -1);
//Either in D or R
if ((gvals[gid] != 1 && gvals[gcid] == 1)
|| (gvals[gid] == 1 && A->LRID(gcid) != -1 && gvals[gcid] == 1))
{
assert(lcnt < data->lmax);
if (insertValues)
LeftIndex[lcnt] = gcid;
else
{
//local column id
lcid = (gvals[gid] == 1 ? D->LCID(gcid) : R->LCID(gcid));
assert(lcid != -1);
LeftIndex[lcnt] = lcid;
}
LeftValues[lcnt++] = Ax[j];
}
else
{
assert(rcnt < data->rmax);
if (insertValues)
RightIndex[rcnt] = gcid;
else
{
//local column id
lcid = (gvals[gid] == 1 ? C->LCID(gcid) : G->LCID(gcid));
assert(lcid != -1);
RightIndex[rcnt] = lcid;
}
RightValues[rcnt++] = Ax[j];
}
}
if (gvals[gid] == 1)
{ // D or C row
if (insertValues)
{
err = D->InsertGlobalValues(gid, lcnt, LeftValues, LeftIndex);
assert(err == 0);
err = C->InsertGlobalValues(gid, rcnt, RightValues, RightIndex);
assert(err == 0);
}
else
{
err = D->ReplaceMyValues(D->LRID(gid), lcnt, LeftValues,
LeftIndex);
assert(err == 0);
err = C->ReplaceMyValues(C->LRID(gid), rcnt, RightValues,
RightIndex);
assert(err == 0);
}
}
else
{ // R or S row
//assert(lcnt > 0); // TODO: Enable this once using narrow sep.
if (insertValues)
{
assert(rcnt > 0);
err = R->InsertGlobalValues(gid, lcnt, LeftValues, LeftIndex);
assert(err == 0);
err = G->InsertGlobalValues(gid, rcnt, RightValues, RightIndex);
assert(err == 0);
if (config->schurApproxMethod == 1)
{
err = Sg->InsertGlobalIndices(gid, rcnt, RightIndex);
assert(err == 0);
}
}
else
{
assert(rcnt > 0);
err = R->ReplaceMyValues(R->LRID(gid), lcnt, LeftValues,
LeftIndex);
assert(err == 0);
err = G->ReplaceMyValues(G->LRID(gid), rcnt, RightValues,
RightIndex);
assert(err == 0);
}
}
}
if (insertValues)
{
/* ------------- Create the maps for the DBBD form ------------------ */
Epetra_Map *DRowMap, *SRowMap, *DColMap;
Epetra_SerialComm LComm;
if (config->sep_type != 1)
{
DRowMap = new Epetra_Map(-1, data->Dnr, data->DRowElems, 0,
A->Comm());
SRowMap = new Epetra_Map(-1, data->Snr, data->SRowElems, 0,
A->Comm());
DColMap = new Epetra_Map(-1, data->Dnc, DColElems, 0,
A->Comm());
}
else
{
DRowMap = new Epetra_Map(-1, data->Dnr, data->DRowElems, 0, LComm);
SRowMap = new Epetra_Map(-1, data->Snr, data->SRowElems, 0, LComm);
DColMap = new Epetra_Map(-1, data->Dnc, DColElems, 0, LComm);
}
D->FillComplete();
//config->dm.print(5, "Done D fillcomplete");
G->FillComplete();
//config->dm.print(5, "Done G fillcomplete");
C->FillComplete(*SRowMap, *DRowMap); //TODO:Won't work if permutation is
// unsymmetric SRowMap
//config->dm.print(5, "Done C fillcomplete");
R->FillComplete(*DColMap, *SRowMap);
//config->dm.print(5, "Done R fillcomplete");
int Sdiag = (int) data->Snr * Sdiagfactor;
Sdiag = MIN(Sdiag, data->Snr-1);
Sdiag = MAX(Sdiag, 0);
// Add the diagonals to Sg
for (int i = 0; config->schurApproxMethod == 1 && i < nrows ; i++)
{
gid = rows[i];
if (gvals[gid] == 1) continue; // not a row in S
if (data->Snr == 0) assert(0 == 1);
rcnt = 0;
//TODO Will be trouble if SNumGlobalCols != Snc
//assert(SNumGlobalCols == Snc);
//for (int j = MAX(i-Sdiag,0) ; j<MIN(SNumGlobalCols, i+Sdiag); j++)
for (int j = MAX(i-Sdiag, 0) ; j < MIN(data->Snr, i+Sdiag); j++)
{
// find the adjacent columns from the row map of S
//assert (j >= 0 && j < Snr);
RightIndex[rcnt++] = data->SRowElems[j];
}
err = Sg->InsertGlobalIndices(gid, rcnt, RightIndex);
assert(err == 0);
// Always insert the diagonals, if it is added twice that is fine.
err = Sg->InsertGlobalIndices(gid, 1, &gid);
assert(err == 0);
}
if (config->schurApproxMethod == 1)
Sg->FillComplete();
delete DRowMap;
delete SRowMap;
delete DColMap;
}
#if 0
if (insertValues)
{
#ifdef TIMING_OUTPUT
Teuchos::Time ttime("transpose time");
ttime.start();
#endif
bool MakeDataContiguous = true;
ssym->transposer = Teuchos::RCP<EpetraExt::RowMatrix_Transpose>(new EpetraExt::RowMatrix_Transpose(MakeDataContiguous));
ssym->DT = Teuchos::rcp( dynamic_cast<Epetra_CrsMatrix *>(&(*ssym->transposer)(*D)), false);
#ifdef TIMING_OUTPUT
ttime.stop();
cout << "Transpose Time" << ttime.totalElapsedTime() << endl;
ttime.reset();
#endif
}
else
{
ssym->transposer->fwd();
//ssym->ReIdx_LP->fwd(); // TODO: Needed ?
}
#endif
// A is no longer needed
delete[] LeftIndex;
delete[] LeftValues;
delete[] RightIndex;
delete[] RightValues;
//cout << msg << "S rows=" << S.NumGlobalRows() << " S cols=" <<
//S.NumGlobalCols() << "#cols in column map="<<
//S.ColMap().NumMyElements() << endl;
//cout << msg << "C rows=" << Cptr->NumGlobalRows() << " C cols=" <<
//Cptr->NumGlobalCols() << "#cols in column map="<<
//Cptr->ColMap().NumMyElements() << endl;
//cout << msg << "D rows=" << D.NumGlobalRows() << " D cols=" <<
//D.NumGlobalCols() << "#cols in column map="<<
//D.ColMap().NumMyElements() << endl;
//cout << msg << "R rows=" << Rptr->NumGlobalRows() << " R cols=" <<
//Rptr->NumGlobalCols() << "#cols in column map="<<
//Rptr->ColMap().NumMyElements() << endl;
// ]
return 0;
}
/* Computes the approximate Schur complement for the wide separator
using guided probing*/
Teuchos::RCP<Epetra_CrsMatrix> computeSchur_GuidedProbing
(
shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure
Epetra_Map *localDRowMap
)
{
int i;
double relative_thres = config->relative_threshold;
Epetra_CrsMatrix *G = ssym->G.getRawPtr();
Epetra_CrsMatrix *R = ssym->R.getRawPtr();
Epetra_LinearProblem *LP = ssym->LP.getRawPtr();
Amesos_BaseSolver *solver = ssym->Solver.getRawPtr();
Ifpack_Preconditioner *ifSolver = ssym->ifSolver.getRawPtr();
Epetra_CrsMatrix *C = ssym->C.getRawPtr();
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
Epetra_Map CrMap = C->RowMap();
int *c_rows = CrMap.MyGlobalElements();
int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
int c_localElems = CrMap.NumMyElements();
int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
Epetra_Map RrMap = R->RowMap();
int *r_rows = RrMap.MyGlobalElements();
int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
int r_localElems = RrMap.NumMyElements();
int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map C_localRMap (-1, c_localElems, c_rows, 0, LComm);
Epetra_Map C_localCMap (-1, c_localcolElems, c_cols, 0, LComm);
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
Epetra_Map R_localRMap (-1, r_localElems, r_rows, 0, LComm);
Epetra_Map R_localCMap (-1, r_localcolElems, r_cols, 0, LComm);
//cout << "#local rows" << g_localElems << "#non zero local cols" << c_localcolElems << endl;
#ifdef DEBUG
cout << "DEBUG MODE" << endl;
int nrows = C->RowMap().NumMyElements();
assert(nrows == localDRowMap->NumGlobalElements());
int gids[nrows], gids1[nrows];
C_localRMap.MyGlobalElements(gids);
localDRowMap->MyGlobalElements(gids1);
cout << "Comparing R's domain map with D's row map" << endl;
for (int i = 0; i < nrows; i++)
{
assert(gids[i] == gids1[i]);
}
#endif
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
//cout << "Creating local matrices" << endl;
int err;
Epetra_CrsMatrix localC(Copy, C_localRMap, C->MaxNumEntries(), false);
for (i = 0; i < c_localElems ; i++)
{
gid = c_rows[i];
err = C->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1,
indices1);
assert (err == 0);
//if (nentries1 > 0) // TODO: Later
//{
err = localC.InsertGlobalValues(gid, nentries1, values1, indices1);
assert (err == 0);
//}
}
localC.FillComplete(G_localRMap, C_localRMap);
//cout << "Created local C matrix" << endl;
Epetra_CrsMatrix localR(Copy, R_localRMap, R->MaxNumEntries(), false);
for (i = 0; i < r_localElems ; i++)
{
gid = r_rows[i];
R->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
localR.InsertGlobalValues(gid, nentries1, values1, indices1);
}
localR.FillComplete(*localDRowMap, R_localRMap);
//cout << "Created local R matrix" << endl;
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
cout << "Created local G matrix" << endl;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, ssym, &localG, &localR, LP, solver,
ifSolver, &localC, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
Teuchos::RCP<Epetra_CrsGraph> lSGraph = Teuchos::RCP<Epetra_CrsGraph> (
new Epetra_CrsGraph(Copy, G_localRMap, maxentries));
if (data->num_compute % config->reset_iter == 0)
{
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[g_localElems];
int *indices = new int[g_localElems];
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec(G_localRMap, nvectors);
Epetra_MultiVector Scol(G_localRMap, nvectors);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
probevec.PutScalar(0.0); // TODO: Move it out
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
// Not much of use for Shasta 2x2 .. Later.
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[cindex] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0)
{
dropped++;
//cout << "vecvalues[j]" << vecvalues[j] <<
// " max" << maxvalue[k] << endl;
}
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < g_localElems ; i++)
{
// TODO: Can move the next two decalarations outside the loop
Epetra_MultiVector probevec(G_localRMap, 1);
Epetra_MultiVector Scol(G_localRMap, 1);
probevec.PutScalar(0.0);
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[i], 0, 1.0);
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[i] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
probeop.PrintTimingInfo();
Sbar->FillComplete();
lSGraph->FillComplete();
data->localSbargraph = lSGraph;
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
cout << "#dropped entries" << dropped << endl;
delete[] values;
delete[] indices;
delete[] maxvalue;
}
else
{
if (((data->num_compute-1) % config->reset_iter) == 0)
{
// We recomputed the Schur complement with dropping for the last
// compute. Reset the prober with the new orthogonal vectors for
// the Sbar from the previous iteration.
Teuchos::ParameterList pList;
Teuchos::RCP<Isorropia::Epetra::Prober> gprober =
Teuchos::RCP<Isorropia::Epetra::Prober> (new
Isorropia::Epetra::Prober(
data->localSbargraph.getRawPtr(), pList, false));
gprober->color();
data->guided_prober = gprober;
}
// Use the prober to probe the probeop for the sparsity pattern
// add that to Sbar and call Fill complete
int nvectors = data->guided_prober->getNumOrthogonalVectors();
cout << "Number of Orthogonal Vectors for guided probing" << nvectors
<< endl;
probeop.ResetTempVectors(nvectors);
Teuchos::RCP<Epetra_CrsMatrix> blockdiag_Sbar =
data->guided_prober->probe(probeop);
int maxentries = blockdiag_Sbar->GlobalMaxNumEntries();
int *indices = new int[maxentries];
double *values = new double[maxentries];
int numentries;
for (int i = 0; i < blockdiag_Sbar->NumGlobalRows() ; i++)
{
int gid = blockdiag_Sbar->GRID(i);
blockdiag_Sbar->ExtractGlobalRowCopy(gid, maxentries, numentries,
values, indices);
Sbar->InsertGlobalValues(gid, numentries, values, indices);
}
Sbar->FillComplete();
delete[] indices;
delete[] values;
}
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
return Sbar;
}
// ***********************************************************
int DG_Prob::Eigenvectors(const double Dt,
const Epetra_Map & Map)
{
printf("Entrou em Eigenvectors\n");
#ifdef HAVE_MPI
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
//MPI::COMM_WORLD.Barrier();
Comm.Barrier();
Teuchos::RCP<Epetra_FECrsMatrix> M = Teuchos::rcp(new Epetra_FECrsMatrix(Copy, Map,0));//&NNz[0]);
Teuchos::RCP<Epetra_FEVector> RHS = Teuchos::rcp(new Epetra_FEVector(Map,1));
DG_MatrizVetor_Epetra(Dt,M,RHS);
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp(new Epetra_CrsMatrix(Copy, Map,0
/* &NNz[0]*/) );
Epetra_Export Exporter(Map,Map);
A->PutScalar(0.0);
A->Export(*(M.ptr()),Exporter,Add);
A->FillComplete();
using std::cout;
// int nx = 5;
bool boolret;
int MyPID = Comm.MyPID();
bool verbose = true;
bool debug = false;
std::string which("LR");
Teuchos::CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("debug","nodebug",&debug,"Print debugging information.");
cmdp.setOption("sort",&which,"Targetted eigenvalues (SM,LM,SR,LR,SI,or LI).");
typedef double ScalarType;
typedef Teuchos::ScalarTraits<ScalarType> SCT;
typedef SCT::magnitudeType MagnitudeType;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<ScalarType,MV> MVT;
typedef Anasazi::OperatorTraits<ScalarType,MV,OP> OPT;
// double rho = 2*nx+1;
// Compute coefficients for discrete convection-diffution operator
// const double one = 1.0;
// int NumEntries, info;
//************************************
// Start the block Arnoldi iteration
//***********************************
//
// Variables used for the Block Krylov Schur Method
//
int nev = 10;
int blockSize = 1;
int numBlocks = 20;
int maxRestarts = 500;
//int stepSize = 5;
double tol = 1e-8;
// Create a sort manager to pass into the block Krylov-Schur solver manager
// --> Make sure the reference-counted pointer is of type Anasazi::SortManager<>
// --> The block Krylov-Schur solver manager uses Anasazi::BasicSort<> by default,
// so you can also pass in the parameter "Which", instead of a sort manager.
Teuchos::RCP<Anasazi::SortManager<MagnitudeType> > MySort =
Teuchos::rcp( new Anasazi::BasicSort<MagnitudeType>( which ) );
// Set verbosity level
int verbosity = Anasazi::Errors + Anasazi::Warnings;
if (verbose) {
verbosity += Anasazi::FinalSummary + Anasazi::TimingDetails;
}
if (debug) {
verbosity += Anasazi::Debug;
}
//
// Create parameter list to pass into solver manager
//
Teuchos::ParameterList MyPL;
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Sort Manager", MySort );
//MyPL.set( "Which", which );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Num Blocks", numBlocks );
MyPL.set( "Maximum Restarts", maxRestarts );
//MyPL.set( "Step Size", stepSize );
MyPL.set( "Convergence Tolerance", tol );
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(Map, blockSize) );
ivec->Random();
// Create the eigenproblem.
Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > MyProblem =
Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(A, ivec) );
// Inform the eigenproblem that the operator A is symmetric
//MyProblem->setHermitian(rho==0.0);
// Set the number of eigenvalues requested
MyProblem->setNEV( nev );
// Inform the eigenproblem that you are finishing passing it information
boolret = MyProblem->setProblem();
if (boolret != true) {
if (verbose && MyPID == 0) {
cout << "Anasazi::BasicEigenproblem::setProblem() returned with error." << endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
// Initialize the Block Arnoldi solver
Anasazi::BlockKrylovSchurSolMgr<double, MV, OP> MySolverMgr(MyProblem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMgr.solve();
if (returnCode != Anasazi::Converged && MyPID==0 && verbose) {
cout << "Anasazi::EigensolverMgr::solve() returned unconverged." << endl;
}
// Get the Ritz values from the eigensolver
std::vector<Anasazi::Value<double> > ritzValues = MySolverMgr.getRitzValues();
// Output computed eigenvalues and their direct residuals
if (verbose && MyPID==0) {
int numritz = (int)ritzValues.size();
cout.setf(std::ios_base::right, std::ios_base::adjustfield);
cout<<endl<< "Computed Ritz Values"<< endl;
if (MyProblem->isHermitian()) {
cout<< std::setw(16) << "Real Part"
<< endl;
cout<<"-----------------------------------------------------------"<<endl;
for (int i=0; i<numritz; i++) {
cout<< std::setw(16) << ritzValues[i].realpart
<< endl;
}
cout<<"-----------------------------------------------------------"<<endl;
}
else {
cout<< std::setw(16) << "Real Part"
<< std::setw(16) << "Imag Part"
<< endl;
cout<<"-----------------------------------------------------------"<<endl;
for (int i=0; i<numritz; i++) {
cout<< std::setw(16) << ritzValues[i].realpart
<< std::setw(16) << ritzValues[i].imagpart
<< endl;
}
cout<<"-----------------------------------------------------------"<<endl;
}
}
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<ScalarType,MV> sol = MyProblem->getSolution();
std::vector<Anasazi::Value<ScalarType> > evals = sol.Evals;
Teuchos::RCP<MV> evecs = sol.Evecs;
std::vector<int> index = sol.index;
int numev = sol.numVecs;
if (numev > 0) {
// Compute residuals.
Teuchos::LAPACK<int,double> lapack;
std::vector<double> normA(numev);
if (MyProblem->isHermitian()) {
// Get storage
Epetra_MultiVector Aevecs(Map,numev);
Teuchos::SerialDenseMatrix<int,double> B(numev,numev);
B.putScalar(0.0);
for (int i=0; i<numev; i++) {B(i,i) = evals[i].realpart;}
// Compute A*evecs
OPT::Apply( *A, *evecs, Aevecs );
// Compute A*evecs - lambda*evecs and its norm
MVT::MvTimesMatAddMv( -1.0, *evecs, B, 1.0, Aevecs );
MVT::MvNorm( Aevecs, normA );
// Scale the norms by the eigenvalue
for (int i=0; i<numev; i++) {
normA[i] /= Teuchos::ScalarTraits<double>::magnitude( evals[i].realpart );
}
} else {
// The problem is non-Hermitian.
int i=0;
std::vector<int> curind(1);
std::vector<double> resnorm(1), tempnrm(1);
Teuchos::RCP<MV> tempAevec;
Teuchos::RCP<const MV> evecr, eveci;
Epetra_MultiVector Aevec(Map,numev);
// Compute A*evecs
OPT::Apply( *A, *evecs, Aevec );
Teuchos::SerialDenseMatrix<int,double> Breal(1,1), Bimag(1,1);
while (i<numev) {
if (index[i]==0) {
// Get a view of the current eigenvector (evecr)
curind[0] = i;
evecr = MVT::CloneView( *evecs, curind );
// Get a copy of A*evecr
tempAevec = MVT::CloneCopy( Aevec, curind );
// Compute A*evecr - lambda*evecr
Breal(0,0) = evals[i].realpart;
MVT::MvTimesMatAddMv( -1.0, *evecr, Breal, 1.0, *tempAevec );
// Compute the norm of the residual and increment counter
MVT::MvNorm( *tempAevec, resnorm );
normA[i] = resnorm[0]/Teuchos::ScalarTraits<MagnitudeType>::magnitude( evals[i].realpart );
i++;
} else {
// Get a view of the real part of the eigenvector (evecr)
curind[0] = i;
evecr = MVT::CloneView( *evecs, curind );
// Get a copy of A*evecr
tempAevec = MVT::CloneCopy( Aevec, curind );
// Get a view of the imaginary part of the eigenvector (eveci)
curind[0] = i+1;
eveci = MVT::CloneView( *evecs, curind );
// Set the eigenvalue into Breal and Bimag
Breal(0,0) = evals[i].realpart;
Bimag(0,0) = evals[i].imagpart;
// Compute A*evecr - evecr*lambdar + eveci*lambdai
MVT::MvTimesMatAddMv( -1.0, *evecr, Breal, 1.0, *tempAevec );
MVT::MvTimesMatAddMv( 1.0, *eveci, Bimag, 1.0, *tempAevec );
MVT::MvNorm( *tempAevec, tempnrm );
// Get a copy of A*eveci
tempAevec = MVT::CloneCopy( Aevec, curind );
// Compute A*eveci - eveci*lambdar - evecr*lambdai
MVT::MvTimesMatAddMv( -1.0, *evecr, Bimag, 1.0, *tempAevec );
MVT::MvTimesMatAddMv( -1.0, *eveci, Breal, 1.0, *tempAevec );
MVT::MvNorm( *tempAevec, resnorm );
// Compute the norms and scale by magnitude of eigenvalue
normA[i] = lapack.LAPY2( tempnrm[i], resnorm[i] ) /
lapack.LAPY2( evals[i].realpart, evals[i].imagpart );
normA[i+1] = normA[i];
i=i+2;
}
}
}
// Output computed eigenvalues and their direct residuals
if (verbose && MyPID==0) {
cout.setf(std::ios_base::right, std::ios_base::adjustfield);
cout<<endl<< "Actual Residuals"<<endl;
if (MyProblem->isHermitian()) {
cout<< std::setw(16) << "Real Part"
<< std::setw(20) << "Direct Residual"<< endl;
cout<<"-----------------------------------------------------------"<<endl;
for (int i=0; i<numev; i++) {
cout<< std::setw(16) << evals[i].realpart
<< std::setw(20) << normA[i] << endl;
}
cout<<"-----------------------------------------------------------"<<endl;
}
else {
cout<< std::setw(16) << "Real Part"
<< std::setw(16) << "Imag Part"
<< std::setw(20) << "Direct Residual"<< endl;
cout<<"-----------------------------------------------------------"<<endl;
for (int i=0; i<numev; i++) {
cout<< std::setw(16) << evals[i].realpart
<< std::setw(16) << evals[i].imagpart
<< std::setw(20) << normA[i] << endl;
}
cout<<"-----------------------------------------------------------"<<endl;
}
}
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return 0;
}
int
main(int argc, char *argv[])
{
using Teuchos::CommandLineProcessor;
using Teuchos::ParameterList;
using Teuchos::RCP;
using Teuchos::rcp;
using std::cout;
using std::endl;
typedef double ST;
typedef Teuchos::ScalarTraits<ST> SCT;
typedef SCT::magnitudeType MT;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Belos::MultiVecTraits<ST,MV> MVT;
typedef Belos::OperatorTraits<ST,MV,OP> OPT;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
const int MyPID = Comm.MyPID();
#else // No EPETRA_MPI
Epetra_SerialComm Comm;
const int MyPID = 0;
#endif // EPETRA_MPI
bool verbose = false, debug = false;
int frequency = -1; // frequency of status test output.
int numrhs = 1; // number of right-hand sides to solve for
int maxiters = -1; // maximum number of iterations allowed per linear system
MT tol = 1.0e-8; // relative residual tolerance
// Define command-line arguments
CommandLineProcessor cmdp(false,true);
cmdp.setOption ("verbose", "quiet", &verbose, "Print messages and results.");
cmdp.setOption ("debug", "nodebug", &debug,
"Print debugging information from the solver.");
cmdp.setOption ("frequency", &frequency,
"Solver's frequency for printing residuals (#iters).");
cmdp.setOption ("tol", &tol,
"Relative residual tolerance used by MINRES solver.");
cmdp.setOption ("num-rhs", &numrhs,
"Number of right-hand sides for which to solve (> 0).");
cmdp.setOption ("max-iters", &maxiters,
"Maximum number of iterations per linear system. -1 means "
"we choose, based on problem size.");
// Parse command-line arguments and fetch values
if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL)
{
std::cout << "Failed to parse command-line arguments!" << std::endl;
std::cout << "End Result: TEST FAILED" << std::endl;
return -1;
}
// **********************************************************************
// ******************Set up the problem to be solved*********************
// construct diagonal matrix
const int NumGlobalElements = 1e2;
const int m = 4; // number of negative eigenvalues
// Create diagonal matrix with n-m positive and m negative eigenvalues.
Epetra_Map epetraMap( NumGlobalElements, 0, Comm );
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix( Copy, epetraMap, 1 ) );
for ( int k=0; k<epetraMap.NumMyElements(); k++ )
{
int GIDk = epetraMap.GID(k);
double val = 2*(GIDk-m) + 1;
TEUCHOS_ASSERT_EQUALITY( 0, A->InsertGlobalValues( GIDk, 1, &val, &GIDk ) );
}
TEUCHOS_ASSERT_EQUALITY( 0, A->FillComplete() );
TEUCHOS_ASSERT_EQUALITY( 0, A->OptimizeStorage() );
//
// Make some (multi)vectors, for use in testing: an exact solution,
// its corresponding right-hand side, and an initial guess.
//
// Make a random exact solution.
Teuchos::RCP<Epetra_MultiVector> X_exact (new Epetra_MultiVector (epetraMap, numrhs));
X_exact->Seed();
MVT::MvRandom (*X_exact);
// Compute the right-hand side as B = A*X.
Teuchos::RCP<Epetra_MultiVector> B = MVT::Clone (*X_exact, numrhs);
OPT::Apply (*A, *X_exact, *B);
// Choose an initial guess of all zeros.
Teuchos::RCP<Epetra_MultiVector> X = MVT::Clone (*X_exact, numrhs);
MVT::MvInit (*X, ST(0.0));
// **********************************************************************
//
// Set parameters that the user asked us to pick intelligently.
//
// Maximum number of iterations: set according to problem size.
// maxiters=numGlobalElements-1 may not always converge, so we
// set to numGlobalElements+1 for good measure.
if (maxiters == -1)
maxiters = NumGlobalElements + 1;
// In a nonverbose test, the frequency should be set to -1, which
// Belos interprets as "no intermediate status output." Override
// whatever the user may have specified.
if (! verbose)
frequency = -1;
// Silently fix a bad frequency value.
else if (frequency < 0 && frequency != -1)
frequency = -1;
// Validate command-line arguments
TEUCHOS_TEST_FOR_EXCEPTION( tol < 0, std::invalid_argument,
"Relative residual tolerance must be nonnegative, but "
"you supplied tol = " << tol << "." );
TEUCHOS_TEST_FOR_EXCEPTION( numrhs < 1, std::invalid_argument,
"MINRES test requires at least one right-hand side, but "
"you set the number of right-hand sides to "
<< numrhs << "." );
TEUCHOS_TEST_FOR_EXCEPTION( maxiters < 1, std::invalid_argument,
"MINRES test requires at least one iteration, but you "
"set the maximum number of iterations to "
<< maxiters << "." );
const bool proc_verbose = verbose && (MyPID==0); /* Only print on the zero processor */
//
// ********Other information used by block solver***********
// *****************(can be user specified)******************
//
//
ParameterList belosList;
belosList.set( "Maximum Iterations", maxiters ); // Maximum number of iterations allowed
belosList.set( "Convergence Tolerance", tol ); // Relative convergence tolerance requested
int verbosity = Belos::Errors + Belos::Warnings;
if (verbose) {
verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
if (frequency > 0)
belosList.set( "Output Frequency", frequency );
}
if (debug) {
verbosity += Belos::Debug;
}
belosList.set( "Verbosity", (int) verbosity );
//
// Construct an unpreconditioned linear problem instance.
//
Belos::LinearProblem< ST, MV, OP > problem (A, X, B);
{
const bool set = problem.setProblem();
TEUCHOS_TEST_FOR_EXCEPTION( set == false, std::logic_error,
"Belos::LinearProblem failed to set up correctly (setP"
"roblem() returned false)! This probably means we imp"
"lemented our test incorrectly." );
}
//
// *******************************************************************
// ****************Start the MINRES iteration*************************
// *******************************************************************
//
// Create an iterative solver manager.
RCP< Belos::SolverManager<ST,MV,OP> > newSolver
= rcp( new Belos::MinresSolMgr<ST,MV,OP>(rcp(&problem,false), rcp(&belosList,false)));
//
// **********Print out information about problem*******************
//
if (proc_verbose) {
cout << endl << endl;
cout << "Dimension of matrix: " << NumGlobalElements << endl;
cout << "Number of right-hand sides: " << numrhs << endl;
cout << "Relative residual tolerance: " << tol << endl;
cout << endl;
}
//
// Perform solve
//
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = newSolver->solve();
} catch (std::exception& e) {
if (MyPID == 0)
cout << "MINRES solver threw an exception: " << e.what() << endl;
throw e;
}
//
// Compute actual residuals.
//
bool badRes = false;
std::vector<double> actual_resids( numrhs );
std::vector<double> rhs_norm( numrhs );
Epetra_MultiVector resid(A->Map(), numrhs);
OPT::Apply( *A, *X, resid );
MVT::MvAddMv( -1.0, resid, 1.0, *B, resid );
MVT::MvNorm( resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
if (proc_verbose) {
std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i=0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
std::cout<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
if (actRes > tol) badRes = true;
}
}
if (ret!=Belos::Converged || badRes) {
if (proc_verbose)
std::cout << std::endl << "End Result: TEST FAILED" << std::endl;
}
//
// Default return value
//
if (proc_verbose)
std::cout << std::endl << "End Result: TEST PASSED" << std::endl;
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
if (ret!=Belos::Converged || badRes)
return -1;
else
return 0;
}
// ======================================================================
void GetPtent(const Epetra_RowMatrix& A, Teuchos::ParameterList& List,
double* thisns, Teuchos::RCP<Epetra_CrsMatrix>& Ptent,
Teuchos::RCP<Epetra_MultiVector>& NextNS, const int domainoffset)
{
const int nsdim = List.get<int>("null space: dimension",-1);
if (nsdim<=0) ML_THROW("null space dimension not given",-1);
const Epetra_Map& rowmap = A.RowMatrixRowMap();
const int mylength = rowmap.NumMyElements();
// wrap nullspace into something more handy
Epetra_MultiVector ns(View,rowmap,thisns,mylength,nsdim);
// vector to hold aggregation information
Epetra_IntVector aggs(rowmap,false);
// aggregation with global aggregate numbering
int naggregates = GetGlobalAggregates(const_cast<Epetra_RowMatrix&>(A),List,thisns,aggs);
// build a domain map for Ptent
// find first aggregate on proc
int firstagg = -1;
int offset = -1;
for (int i=0; i<mylength; ++i)
if (aggs[i]>=0)
{
offset = firstagg = aggs[i];
break;
}
offset *= nsdim;
if (offset<0) ML_THROW("could not find any aggregate on proc",-2);
std::vector<int> coarsegids(naggregates*nsdim);
for (int i=0; i<naggregates; ++i)
for (int j=0; j<nsdim; ++j)
{
coarsegids[i*nsdim+j] = offset + domainoffset;
++offset;
}
Epetra_Map pdomainmap(-1,naggregates*nsdim,&coarsegids[0],0,A.Comm());
// loop aggregates and build ids for dofs
std::map<int,std::vector<int> > aggdofs;
std::map<int,std::vector<int> >::iterator fool;
for (int i=0; i<naggregates; ++i)
{
std::vector<int> gids(0);
aggdofs.insert(std::pair<int,std::vector<int> >(firstagg+i,gids));
}
for (int i=0; i<mylength; ++i)
{
if (aggs[i]<0) continue;
std::vector<int>& gids = aggdofs[aggs[i]];
gids.push_back(aggs.Map().GID(i));
}
#if 0 // debugging output
for (int proc=0; proc<A.Comm().NumProc(); ++proc)
{
if (proc==A.Comm().MyPID())
{
for (fool=aggdofs.begin(); fool!=aggdofs.end(); ++fool)
{
std::cout << "Proc " << proc << " Aggregate " << fool->first << " Dofs ";
std::vector<int>& gids = fool->second;
for (int i=0; i<(int)gids.size(); ++i) std::cout << gids[i] << " ";
std::cout << std::endl;
}
}
fflush(stdout);
A.Comm().Barrier();
}
#endif
// coarse level nullspace to be filled
NextNS = Teuchos::rcp(new Epetra_MultiVector(pdomainmap,nsdim,true));
Epetra_MultiVector& nextns = *NextNS;
// matrix Ptent
Ptent = Teuchos::rcp(new Epetra_CrsMatrix(Copy,rowmap,nsdim));
// loop aggregates and extract the appropriate slices of the nullspace.
// do QR and assemble Q and R to Ptent and NextNS
for (fool=aggdofs.begin(); fool!=aggdofs.end(); ++fool)
{
// extract aggregate-local junk of nullspace
const int aggsize = (int)fool->second.size();
Epetra_SerialDenseMatrix Bagg(aggsize,nsdim);
for (int i=0; i<aggsize; ++i)
for (int j=0; j<nsdim; ++j)
Bagg(i,j) = (*ns(j))[ns.Map().LID(fool->second[i])];
// Bagg = Q*R
int m = Bagg.M();
int n = Bagg.N();
int lwork = n*10;
int info = 0;
int k = std::min(m,n);
if (k!=n) ML_THROW("Aggregate too small, fatal!",-1);
std::vector<double> work(lwork);
std::vector<double> tau(k);
Epetra_LAPACK lapack;
lapack.GEQRF(m,n,Bagg.A(),m,&tau[0],&work[0],lwork,&info);
if (info) ML_THROW("Lapack dgeqrf returned nonzero",info);
if (work[0]>lwork)
{
lwork = (int)work[0];
work.resize(lwork);
}
// get R (stored on Bagg) and assemble it into nextns
int agg_cgid = fool->first*nsdim;
if (!nextns.Map().MyGID(agg_cgid+domainoffset))
ML_THROW("Missing coarse column id on this proc",-1);
for (int i=0; i<n; ++i)
for (int j=i; j<n; ++j)
(*nextns(j))[nextns.Map().LID(domainoffset+agg_cgid+i)] = Bagg(i,j);
// get Q and assemble it into Ptent
lapack.ORGQR(m,n,k,Bagg.A(),m,&tau[0],&work[0],lwork,&info);
if (info) ML_THROW("Lapack dorgqr returned nonzero",info);
for (int i=0; i<aggsize; ++i)
{
const int actgrow = fool->second[i];
for (int j=0; j<nsdim; ++j)
{
int actgcol = fool->first*nsdim+j+domainoffset;
int errone = Ptent->SumIntoGlobalValues(actgrow,1,&Bagg(i,j),&actgcol);
if (errone>0)
{
int errtwo = Ptent->InsertGlobalValues(actgrow,1,&Bagg(i,j),&actgcol);
if (errtwo<0) ML_THROW("Epetra_CrsMatrix::InsertGlobalValues returned negative nonzero",errtwo);
}
else if (errone) ML_THROW("Epetra_CrsMatrix::SumIntoGlobalValues returned negative nonzero",errone);
}
} // for (int i=0; i<aggsize; ++i)
} // for (fool=aggdofs.begin(); fool!=aggdofs.end(); ++fool)
int err = Ptent->FillComplete(pdomainmap,rowmap);
if (err) ML_THROW("Epetra_CrsMatrix::FillComplete returned nonzero",err);
err = Ptent->OptimizeStorage();
if (err) ML_THROW("Epetra_CrsMatrix::OptimizeStorage returned nonzero",err);
return;
}
//---------------------------------------------------------------------------//
TEUCHOS_UNIT_TEST( UniformForwardSource, nh_set )
{
typedef Epetra_Vector VectorType;
typedef MCLS::VectorTraits<VectorType> VT;
typedef Epetra_RowMatrix MatrixType;
typedef MCLS::MatrixTraits<VectorType,MatrixType> MT;
typedef MCLS::ForwardHistory<int> HistoryType;
typedef std::mt19937 rng_type;
typedef MCLS::ForwardDomain<VectorType,MatrixType,rng_type> DomainType;
Teuchos::RCP<const Teuchos::Comm<int> > comm =
Teuchos::DefaultComm<int>::getComm();
Teuchos::RCP<Epetra_Comm> epetra_comm = getEpetraComm( comm );
int comm_size = comm->getSize();
int local_num_rows = 10;
int global_num_rows = local_num_rows*comm_size;
Teuchos::RCP<Epetra_Map> map = Teuchos::rcp(
new Epetra_Map( global_num_rows, 0, *epetra_comm ) );
// Build the linear system.
Teuchos::RCP<Epetra_CrsMatrix> A =
Teuchos::rcp( new Epetra_CrsMatrix( Copy, *map, 0 ) );
Teuchos::Array<int> global_columns( 1 );
Teuchos::Array<double> values( 1 );
for ( int i = 1; i < global_num_rows; ++i )
{
global_columns[0] = i-1;
values[0] = -0.5;
A->InsertGlobalValues( i, global_columns().size(),
&values[0], &global_columns[0] );
}
global_columns[0] = global_num_rows-1;
values[0] = -0.5;
A->InsertGlobalValues( global_num_rows-1, global_columns().size(),
&values[0], &global_columns[0] );
A->FillComplete();
Teuchos::RCP<MatrixType> A_T = MT::copyTranspose(*A);
Teuchos::RCP<VectorType> x = MT::cloneVectorFromMatrixRows( *A );
Teuchos::RCP<VectorType> b = MT::cloneVectorFromMatrixRows( *A );
VT::putScalar( *b, -1.0 );
// Build the forward domain.
Teuchos::ParameterList plist;
plist.set<int>( "Overlap Size", 0 );
Teuchos::RCP<DomainType> domain = Teuchos::rcp( new DomainType( A_T, x, plist ) );
// History setup.
HistoryType::setByteSize();
// Create the forward source with a set number of histories.
int mult = 10;
double cutoff = 1.0e-8;
plist.set<double>("Sample Ratio",mult);
plist.set<double>("Weight Cutoff", cutoff);
MCLS::UniformForwardSource<DomainType>
source( b, domain, comm, comm->getSize(), comm->getRank(), plist );
TEST_ASSERT( source.empty() );
TEST_EQUALITY( source.numToTransport(), 0 );
TEST_EQUALITY( source.numToTransportInSet(), mult*global_num_rows );
TEST_EQUALITY( source.numRequested(), mult*global_num_rows );
TEST_EQUALITY( source.numLeft(), 0 );
TEST_EQUALITY( source.numEmitted(), 0 );
// Build the source.
source.buildSource();
TEST_ASSERT( !source.empty() );
TEST_EQUALITY( source.numToTransport(), mult*local_num_rows );
TEST_EQUALITY( source.numToTransportInSet(), mult*global_num_rows );
TEST_EQUALITY( source.numRequested(), mult*global_num_rows );
TEST_EQUALITY( source.numLeft(), mult*local_num_rows );
TEST_EQUALITY( source.numEmitted(), 0 );
// Sample the source.
Teuchos::RCP<MCLS::PRNG<rng_type> > rng = Teuchos::rcp(
new MCLS::PRNG<rng_type>(comm->getRank()) );
source.setRNG( rng );
for ( int i = 0; i < mult*local_num_rows; ++i )
{
TEST_ASSERT( !source.empty() );
TEST_EQUALITY( source.numLeft(), mult*local_num_rows-i );
TEST_EQUALITY( source.numEmitted(), i );
Teuchos::RCP<HistoryType> history = source.getHistory();
TEST_EQUALITY( history->weight(), 1.0 );
TEST_ASSERT( domain->isGlobalState( history->globalState() ) );
TEST_ASSERT( history->alive() );
TEST_ASSERT( VT::isGlobalRow( *x, history->globalState() ) );
}
TEST_ASSERT( source.empty() );
TEST_EQUALITY( source.numLeft(), 0 );
TEST_EQUALITY( source.numEmitted(), mult*local_num_rows );
}
int main(int argc, char *argv[])
{
int i;
bool ierr, gerr;
gerr = true;
#ifdef HAVE_MPI
// Initialize MPI and setup an Epetra communicator
MPI_Init(&argc,&argv);
Teuchos::RCP<Epetra_MpiComm> Comm = Teuchos::rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
// If we aren't using MPI, then setup a serial communicator.
Teuchos::RCP<Epetra_SerialComm> Comm = Teuchos::rcp( new Epetra_SerialComm() );
#endif
// number of global elements
const int dim = 100;
const int blockSize = 5;
bool verbose = false;
if (argc>1) {
if (argv[1][0]=='-' && argv[1][1]=='v') {
verbose = true;
}
}
// Create an output manager to handle the I/O from the solver
Teuchos::RCP<Anasazi::OutputManager<double> > MyOM = Teuchos::rcp( new Anasazi::BasicOutputManager<double>() );
if (verbose) {
MyOM->setVerbosity( Anasazi::Warnings );
}
#ifndef HAVE_EPETRA_THYRA
MyOM->stream(Anasazi::Warnings)
<< "Please configure Anasazi with:" << std::endl
<< "--enable-epetra-thyra" << std::endl
<< "--enable-anasazi-thyra" << std::endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return -1;
#endif
// Construct a Map that puts approximately the same number of
// equations on each processor.
Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( new Epetra_Map(dim, 0, *Comm) );
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map->NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map->MyGlobalElements(&MyGlobalElements[0]);
// Create an integer vector NumNz that is used to build the Petra Matrix.
// NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation
// on this processor
std::vector<int> NumNz(NumMyElements);
// We are building a tridiagonal matrix where each row has (-1 2 -1)
// So we need 2 off-diagonal terms (except for the first and last equation)
for (i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i]==0 || MyGlobalElements[i] == dim-1) {
NumNz[i] = 2;
}
else {
NumNz[i] = 3;
}
}
// Create an Epetra_Matrix
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix(Copy, *Map, &NumNz[0]) );
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
double two = 2.0;
int NumEntries;
for (i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == dim-1) {
Indices[0] = dim-2;
NumEntries = 1;
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
ierr = A->InsertGlobalValues(MyGlobalElements[i],NumEntries,&Values[0],&Indices[0]);
assert(ierr==0);
// Put in the diagonal entry
ierr = A->InsertGlobalValues(MyGlobalElements[i],1,&two,&MyGlobalElements[i]);
assert(ierr==0);
}
// Finish building the epetra matrix A
ierr = A->FillComplete();
assert(ierr==0);
#ifdef HAVE_EPETRA_THYRA
typedef Thyra::MultiVectorBase<double> TMVB;
typedef Thyra::LinearOpBase<double> TLOB;
// first, create a Thyra::VectorSpaceBase from an Epetra_Map using the Epetra-Thyra wrappers
Teuchos::RCP<const Thyra::VectorSpaceBase<double> > space = Thyra::create_VectorSpace(Map);
// then, create a Thyra::MultiVectorBase from the Thyra::VectorSpaceBase using Thyra creational functions
Teuchos::RCP<Thyra::MultiVectorBase<double> > thyra_ivec = Thyra::createMembers(space,blockSize);
// then, create a Thyra::LinearOpBase from the Epetra_CrsMatrix using the Epetra-Thyra wrappers
Teuchos::RCP<const Thyra::LinearOpBase<double> > thyra_op = Thyra::epetraLinearOp(A);
// test the Thyra multivector adapter
ierr = Anasazi::TestMultiVecTraits<double,TMVB>(MyOM,thyra_ivec);
gerr |= ierr;
if (ierr) {
MyOM->stream(Anasazi::Warnings) << "*** ThyraAdapter PASSED TestMultiVecTraits()" << std::endl;
}
else {
MyOM->stream(Anasazi::Warnings) << "*** ThyraAdapter FAILED TestMultiVecTraits() ***" << std::endl << std::endl;
}
// test the Thyra operator adapter
ierr = Anasazi::TestOperatorTraits<double,TMVB,TLOB>(MyOM,thyra_ivec,thyra_op);
gerr |= ierr;
if (ierr) {
MyOM->stream(Anasazi::Warnings) << "*** ThyraAdapter PASSED TestOperatorTraits()" << std::endl;
}
else {
MyOM->stream(Anasazi::Warnings) << "*** ThyraAdapter FAILED TestOperatorTraits() ***" << std::endl << std::endl;
}
#endif
#ifdef HAVE_MPI
MPI_Finalize();
#endif
if (gerr == false) {
MyOM->print(Anasazi::Warnings,"End Result: TEST FAILED\n");
return -1;
}
//
// Default return value
//
MyOM->print(Anasazi::Warnings,"End Result: TEST PASSED\n");
return 0;
}
int main(int argc, char *argv[])
{
using Teuchos::rcp_implicit_cast;
int i, ierr, gerr;
gerr = 0;
#ifdef HAVE_MPI
// Initialize MPI and setup an Epetra communicator
MPI_Init(&argc,&argv);
Teuchos::RCP<Epetra_MpiComm> Comm = Teuchos::rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
// If we aren't using MPI, then setup a serial communicator.
Teuchos::RCP<Epetra_SerialComm> Comm = Teuchos::rcp( new Epetra_SerialComm() );
#endif
// number of global elements
int dim = 100;
int blockSize = 3;
// PID info
int MyPID = Comm->MyPID();
bool verbose = 0;
if (argc>1) {
if (argv[1][0]=='-' && argv[1][1]=='v') {
verbose = true;
}
}
// Construct a Map that puts approximately the same number of
// equations on each processor.
Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( new Epetra_Map(dim, 0, *Comm) );
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map->NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map->MyGlobalElements(&MyGlobalElements[0]);
// Create an integer std::vector NumNz that is used to build the Petra Matrix.
// NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation
// on this processor
std::vector<int> NumNz(NumMyElements);
// We are building a tridiagonal matrix where each row has (-1 2 -1)
// So we need 2 off-diagonal terms (except for the first and last equation)
for (i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i]==0 || MyGlobalElements[i] == dim-1) {
NumNz[i] = 2;
}
else {
NumNz[i] = 3;
}
}
// Create an Epetra_Matrix
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix(Copy, *Map, &NumNz[0]) );
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
double two = 2.0;
int NumEntries;
for (i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == dim-1) {
Indices[0] = dim-2;
NumEntries = 1;
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
ierr = A->InsertGlobalValues(MyGlobalElements[i],NumEntries,&Values[0],&Indices[0]);
assert(ierr==0);
// Put in the diagonal entry
ierr = A->InsertGlobalValues(MyGlobalElements[i],1,&two,&MyGlobalElements[i]);
assert(ierr==0);
}
// Finish building the epetra matrix A
ierr = A->FillComplete();
assert(ierr==0);
// Create an Belos::EpetraOp from this Epetra_CrsMatrix
Teuchos::RCP<Belos::EpetraOp> op = Teuchos::rcp(new Belos::EpetraOp(A));
// Issue several useful typedefs;
typedef Belos::MultiVec<double> EMV;
typedef Belos::Operator<double> EOP;
// Create an Epetra_MultiVector for an initial std::vector to start the solver.
// Note that this needs to have the same number of columns as the blocksize.
Teuchos::RCP<Belos::EpetraMultiVec> ivec = Teuchos::rcp( new Belos::EpetraMultiVec(*Map, blockSize) );
ivec->Random();
// Create an output manager to handle the I/O from the solver
Teuchos::RCP<Belos::OutputManager<double> > MyOM = Teuchos::rcp( new Belos::OutputManager<double>( MyPID ) );
if (verbose) {
MyOM->setVerbosity( Belos::Errors + Belos::Warnings );
}
#ifdef HAVE_EPETRA_THYRA
typedef Thyra::MultiVectorBase<double> TMVB;
typedef Thyra::LinearOpBase<double> TLOB;
// create thyra objects from the epetra objects
// first, a Thyra::VectorSpaceBase
Teuchos::RCP<const Thyra::VectorSpaceBase<double> > epetra_vs =
Thyra::create_VectorSpace(Map);
// then, a MultiVectorBase (from the Epetra_MultiVector)
Teuchos::RCP<Thyra::MultiVectorBase<double> > thyra_ivec =
Thyra::create_MultiVector(rcp_implicit_cast<Epetra_MultiVector>(ivec),epetra_vs);
// then, a LinearOpBase (from the Epetra_CrsMatrix)
Teuchos::RCP<Thyra::LinearOpBase<double> > thyra_op =
Teuchos::rcp( new Thyra::EpetraLinearOp(A) );
// test the Thyra adapter multivector
ierr = Belos::TestMultiVecTraits<double,TMVB>(MyOM,thyra_ivec);
gerr |= ierr;
switch (ierr) {
case Belos::Ok:
if ( verbose && MyPID==0 ) {
std::cout << "*** ThyraAdapter PASSED TestMultiVecTraits()" << std::endl;
}
break;
case Belos::Error:
if ( verbose && MyPID==0 ) {
std::cout << "*** ThyraAdapter FAILED TestMultiVecTraits() ***"
<< std::endl << std::endl;
}
break;
}
// test the Thyra adapter operator
ierr = Belos::TestOperatorTraits<double,TMVB,TLOB>(MyOM,thyra_ivec,thyra_op);
gerr |= ierr;
switch (ierr) {
case Belos::Ok:
if ( verbose && MyPID==0 ) {
std::cout << "*** ThyraAdapter PASSED TestOperatorTraits()" << std::endl;
}
break;
case Belos::Error:
if ( verbose && MyPID==0 ) {
std::cout << "*** ThyraAdapter FAILED TestOperatorTraits() ***"
<< std::endl << std::endl;
}
break;
}
#endif
#ifdef HAVE_MPI
MPI_Finalize();
#endif
if (gerr) {
if (verbose && MyPID==0)
std::cout << "End Result: TEST FAILED" << std::endl;
return -1;
}
//
// Default return value
//
if (verbose && MyPID==0)
std::cout << "End Result: TEST PASSED" << std::endl;
return 0;
}
Teuchos::RCP<Epetra_CrsGraph> BlockAdjacencyGraph::compute( Epetra_CrsGraph& B, int nbrr, std::vector<int>&r, std::vector<double>& weights, bool verbose)
{
// Check if the graph is on one processor.
int myMatProc = -1, matProc = -1;
int myPID = B.Comm().MyPID();
for (int proc=0; proc<B.Comm().NumProc(); proc++)
{
if (B.NumGlobalEntries() == B.NumMyEntries())
myMatProc = myPID;
}
B.Comm().MaxAll( &myMatProc, &matProc, 1 );
if( matProc == -1)
{ cout << "FAIL for Global! All CrsGraph entries must be on one processor!\n"; abort(); }
int i= 0, j = 0, k, l = 0, p, pm, q = -1, ns;
int tree_height;
int error = -1; /* error detected, possibly a problem with the input */
int nrr; /* number of rows in B */
int nzM = 0; /* number of edges in graph */
int m = 0; /* maximum number of nonzeros in any block row of B */
int* colstack = 0; /* stack used to process each block row */
int* bstree = 0; /* binary search tree */
std::vector<int> Mi, Mj, Mnum(nbrr+1,0);
nrr = B.NumMyRows();
if ( matProc == myPID && verbose )
std::printf(" Matrix Size = %d Number of Blocks = %d\n",nrr, nbrr);
else
nrr = -1; /* Prevent processor from doing any computations */
bstree = csr_bst(nbrr); /* 0 : nbrr-1 */
tree_height = ceil31log2(nbrr) + 1;
error = -1;
l = 0; j = 0; m = 0;
for( i = 0; i < nrr; i++ ){
if( i >= r[l+1] ){
++l; /* new block row */
m = EPETRA_MAX(m,j) ; /* nonzeros in block row */
j = B.NumGlobalIndices(i);
}else{
j += B.NumGlobalIndices(i);
}
}
/* one more time for the final block */
m = EPETRA_MAX(m,j) ; /* nonzeros in block row */
colstack = (int*) malloc( EPETRA_MAX(m,1) * sizeof(int) );
// The compressed graph is actually computed twice,
// due to concerns about memory limitations. First,
// without memory allocation, just nzM is computed.
// Next Mj is allocated. Then, the second time, the
// arrays are actually populated.
nzM = 0; q = -1; l = 0;
int * indices;
int numEntries;
for( i = 0; i <= nrr; i++ ){
if( i >= r[l+1] ){
if( q > 0 ) std::qsort(colstack,q+1,sizeof(int),compare_ints); /* sort stack */
if( q >= 0 ) ns = 1; /* l, colstack[0] M */
for( j=1; j<=q ; j++ ){ /* delete copies */
if( colstack[j] > colstack[j-1] ) ++ns;
}
nzM += ns; /*M->p[l+1] = M->p[l] + ns;*/
++l;
q = -1;
}
if( i < nrr ){
B.ExtractMyRowView( i, numEntries, indices );
for( k = 0; k < numEntries; k++){
j = indices[k]; ns = 0; p = 0;
while( (r[bstree[p]] > j) || (j >= r[bstree[p]+1]) ){
if( r[bstree[p]] > j){
p = 2*p+1;
}else{
if( r[bstree[p]+1] <= j) p = 2*p+2;
}
++ns;
if( p > nbrr || ns > tree_height ) {
error = j;
std::printf("error: p %d nbrr %d ns %d %d\n",p,nbrr,ns,j); break;
}
}
colstack[++q] = bstree[p];
}
//if( error >-1 ){ std::printf("%d\n",error); break; }
// p > nbrr is a fatal error that is ignored
}
}
if ( matProc == myPID && verbose )
std::printf("nzM = %d \n", nzM );
Mi.resize( nzM );
Mj.resize( nzM );
q = -1; l = 0; pm = -1;
for( i = 0; i <= nrr; i++ ){
if( i >= r[l+1] ){
if( q > 0 ) std::qsort(colstack,q+1,sizeof(colstack[0]),compare_ints); /* sort stack */
if( q >= 0 ){
Mi[++pm] = l;
Mj[pm] = colstack[0];
}
for( j=1; j<=q ; j++ ){ /* delete copies */
if( colstack[j] > colstack[j-1] ){ /* l, colstack[j] */
Mi[++pm] = l;
Mj[pm] = colstack[j];
}
}
++l;
Mnum[l] = pm + 1;
/* sparse row format: M->p[l+1] = M->p[l] + ns; */
q = -1;
}
if( i < nrr ){
B.ExtractMyRowView( i, numEntries, indices );
for( k = 0; k < numEntries; k++){
j = indices[k]; ns = 0; p = 0;
while( (r[bstree[p]] > j) || (j >= r[bstree[p]+1]) ){
if( r[bstree[p]] > j){
p = 2*p+1;
}else{
if( r[bstree[p]+1] <= j) p = 2*p+2;
}
++ns;
}
colstack[++q] = bstree[p];
}
}
}
if ( bstree ) free ( bstree );
if ( colstack ) free( colstack );
// Compute weights as number of rows in each block.
weights.resize( nbrr );
for( l=0; l<nbrr; l++) weights[l] = r[l+1] - r[l];
// Compute Epetra_CrsGraph and return
Teuchos::RCP<Epetra_Map> newMap;
if ( matProc == myPID )
newMap = Teuchos::rcp( new Epetra_Map(nbrr, nbrr, 0, B.Comm() ) );
else
newMap = Teuchos::rcp( new Epetra_Map( nbrr, 0, 0, B.Comm() ) );
Teuchos::RCP<Epetra_CrsGraph> newGraph = Teuchos::rcp( new Epetra_CrsGraph( Copy, *newMap, 0 ) );
for( l=0; l<newGraph->NumMyRows(); l++) {
newGraph->InsertGlobalIndices( l, Mnum[l+1]-Mnum[l], &Mj[Mnum[l]] );
}
newGraph->FillComplete();
return (newGraph);
}
static int run_test(Teuchos::RCP<Epetra_CrsMatrix> matrix,
bool verbose, // display the graph before & after
bool contract, // set global number of partitions to 1/2 num procs
int partitioningType, // hypergraph or graph partitioning, or simple
int vertexWeightType, // use vertex weights?
int edgeWeightType, // use edge/hyperedge weights?
int objectType) // use isorropia's CrsMatrix or CrsGraph
{
int rc=0, fail = 0;
#ifdef HAVE_EPETRAEXT
int localProc = 0;
double balance1, balance2, cutn1, cutn2, cutl1, cutl2;
double balance3, cutn3, cutl3;
double cutWgt1, cutWgt2, cutWgt3;
int numCuts1, numCuts2, numCuts3, valid;
int numPartitions = 0;
int keepDenseEdges = 0;
int numProcs = 1;
#ifdef HAVE_MPI
const Epetra_MpiComm &Comm = dynamic_cast<const Epetra_MpiComm &>(matrix->Comm());
localProc = Comm.MyPID();
numProcs = Comm.NumProc();
#else
const Epetra_SerialComm &Comm = dynamic_cast<const Epetra_SerialComm &>(matrix->Comm());
#endif
int numRows = matrix->NumGlobalRows();
if (numRows < (numProcs * 100)){
// By default Zoltan throws out dense edges, defined as those
// whose number of non-zeros exceeds 25% of the number of vertices.
//
// If dense edges are thrown out of a small matrix, there may be nothing left.
keepDenseEdges = 1;
}
double myShareBefore = 1.0 / numProcs;
double myShare = myShareBefore;
if (contract){
numPartitions = numProcs / 2;
if (numPartitions > numRows)
numPartitions = numRows;
if (numPartitions > 0){
if (localProc < numPartitions){
myShare = 1.0 / numPartitions;
}
else{
myShare = 0.0;
}
}
else{
contract = 0;
}
}
// If we want Zoltan's or Isorropia's default weights, then we don't
// need to supply a CostDescriber object to createBalancedCopy,
// so we get to test the API functions that don't take a CostDescriber.
bool noCosts = ((vertexWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS) &&
(edgeWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS));
// Test the interface that has no parameters, if possible
bool noParams =
((partitioningType == HYPERGRAPH_PARTITIONING) && // default, so requires no params
(numPartitions == 0) && // >0 would require a parameter
(keepDenseEdges == 0)); // >0 would require a parameter
// Maps for original object
const Epetra_Map &sourceRowMap = matrix->RowMap();
const Epetra_Map &sourceRangeMap = matrix->RangeMap();
// const Epetra_Map &sourceColMap = matrix->ColMap();
const Epetra_Map &sourceDomainMap = matrix->DomainMap();
int numCols = matrix->NumGlobalCols();
int nMyRows = sourceRowMap.NumMyElements();
int base = sourceRowMap.IndexBase();
// Compute vertex and edge weights
Isorropia::Epetra::CostDescriber costs;
Teuchos::RCP<Epetra_Vector> vptr;
Teuchos::RCP<Epetra_CrsMatrix> eptr;
Teuchos::RCP<Epetra_Vector> hyperEdgeWeights;
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
// Create graph edge weights.
eptr = Teuchos::rcp(new Epetra_CrsMatrix(*matrix));
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
eptr->PutScalar(1.0); // set all nonzeros to 1.0
}
else{
int maxRowSize = eptr->MaxNumEntries();
double *newVal = NULL;
if (maxRowSize > 0){
newVal = new double [maxRowSize];
for (int j=0; j<maxRowSize; j++){
newVal[j] = localProc + 1 + j;
}
}
int numEntries;
int *idx;
double *val;
for (int i=0; i<nMyRows; i++){
rc = eptr->ExtractMyRowView(i, numEntries, val, idx);
for (int j=0; j<numEntries; j++){
val[j] = newVal[j];
}
}
if (newVal) delete [] newVal;
}
eptr->FillComplete(sourceDomainMap, sourceRangeMap);
costs.setGraphEdgeWeights(eptr);
}
else{
// Create hyperedge weights. (Note that the list of hyperedges that a
// process provides weights for has no relation to the columns
// that it has non-zeroes for, or the rows that is has. Hypergraphs
// in general are not square. Also more than one process can provide
// a weight for the same edge. Zoltan combines the weights according
// to the value of the PHG_EDGE_WEIGHT_OPERATION parameter. The default
// for this parameter is to use the maximum edge weight provided by any
// process for a given hyperedge.)
Epetra_Map hyperEdgeMap(numCols, base, Comm);
hyperEdgeWeights = Teuchos::rcp(new Epetra_Vector(hyperEdgeMap));
int *edgeGIDs = NULL;
double *weights = NULL;
int numHEweights = hyperEdgeMap.NumMyElements();
if (numHEweights){
edgeGIDs = new int [numHEweights];
weights = new double [numHEweights];
if (edgeWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
weights[i] = 1.0;
}
}
else{
int hiVolumeStart = matrix->NumGlobalCols() / 3;
int hiVolumeEnd = hiVolumeStart * 2;
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
if ((edgeGIDs[i] < hiVolumeStart) || (edgeGIDs[i] >= hiVolumeEnd)){
weights[i] = 1.0;
}
else{
weights[i] = 3.0;
}
}
}
hyperEdgeWeights->ReplaceGlobalValues(numHEweights, weights, edgeGIDs);
}
if (weights){
delete [] weights;
delete [] edgeGIDs;
}
costs.setHypergraphEdgeWeights(hyperEdgeWeights);
}
}
bool need_importer = false;
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
need_importer = true; // to redistribute row weights
double *val = NULL;
if (nMyRows){
val = new double [nMyRows];
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0;
}
}
else if (vertexWeightType == SUPPLY_UNEQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0 + ((localProc+1) / 2);
}
}
}
vptr = Teuchos::rcp(new Epetra_Vector(Copy, sourceRowMap, val));
if (val) delete [] val;
costs.setVertexWeights(vptr);
}
// Calculate partition quality metrics before calling Zoltan
if (partitioningType == GRAPH_PARTITIONING){
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShare, balance1, numCuts1, cutWgt1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, numCuts3, cutWgt3, cutn3, cutl3);
}
}
else{
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShare, balance1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, cutn3, cutl3);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
Teuchos::ParameterList params;
#ifdef HAVE_ISORROPIA_ZOLTAN
if (!noParams){
// We're using Zoltan for partitioning and supplying
// parameters, overriding defaults.
Teuchos::ParameterList &sublist = params.sublist("Zoltan");
if (partitioningType == GRAPH_PARTITIONING){
params.set("PARTITIONING METHOD", "GRAPH");
sublist.set("GRAPH_PACKAGE", "PHG");
}
else{
params.set("PARTITIONING METHOD", "HYPERGRAPH");
sublist.set("LB_APPROACH", "PARTITION");
sublist.set("PHG_CUT_OBJECTIVE", "CONNECTIVITY"); // "cutl"
}
if (keepDenseEdges){
// only throw out rows that have no zeroes, default is to
// throw out if .25 or more of the columns are non-zero
sublist.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
}
if (numPartitions > 0){
// test #Partitions < #Processes
std::ostringstream os;
os << numPartitions;
std::string s = os.str();
// sublist.set("NUM_GLOBAL_PARTS", s);
params.set("NUM PARTS", s);
}
//sublist.set("DEBUG_LEVEL", "1"); // Zoltan will print out parameters
//sublist.set("DEBUG_LEVEL", "5"); // proc 0 will trace Zoltan calls
//sublist.set("DEBUG_MEMORY", "2"); // Zoltan will trace alloc & free
}
#else
ERROREXIT((localProc==0),
"Zoltan partitioning required but Zoltan not available.")
#endif
// Function scope values
Teuchos::RCP<Epetra_Vector> newvwgts;
Teuchos::RCP<Epetra_CrsMatrix> newewgts;
// Function scope values required for LinearProblem
Epetra_LinearProblem *problem = NULL;
Epetra_Map *LHSmap = NULL;
Epetra_MultiVector *RHS = NULL;
Epetra_MultiVector *LHS = NULL;
// Reference counted pointer to balanced object
Epetra_CrsMatrix *matrixPtr=NULL;
Epetra_CrsGraph *graphPtr=NULL;
Epetra_RowMatrix *rowMatrixPtr=NULL;
Epetra_LinearProblem *problemPtr=NULL;
// Row map for balanced object
const Epetra_BlockMap *targetBlockRowMap=NULL; // for input CrsGraph
const Epetra_Map *targetRowMap=NULL; // for all other inputs
// Column map for balanced object
const Epetra_BlockMap *targetBlockColMap=NULL; // for input CrsGraph
const Epetra_Map *targetColMap=NULL; // for all other inputs
if (objectType == EPETRA_CRSMATRIX){
if (noParams && noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(matrixPtr->RowMap());
targetColMap = &(matrixPtr->ColMap());
}
else if (objectType == EPETRA_CRSGRAPH){
const Epetra_CrsGraph graph = matrix->Graph();
if (noParams && noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph);
}
else if (noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph, params);
}
targetBlockRowMap = &(graphPtr->RowMap());
targetBlockColMap = &(graphPtr->ColMap());
}
else if (objectType == EPETRA_ROWMATRIX){
if (noParams && noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(rowMatrixPtr->RowMatrixRowMap());
targetColMap = &(rowMatrixPtr->RowMatrixColMap());
}
else if (objectType == EPETRA_LINEARPROBLEM){
// Create a linear problem with this matrix.
LHSmap = new Epetra_Map(numCols, base, Comm);
int myRHSsize = sourceRowMap.NumMyElements();
int myLHSsize = LHSmap->NumMyElements();
int valSize = ((myRHSsize > myLHSsize) ? myRHSsize : myLHSsize);
double *vals = NULL;
if (valSize){
vals = new double [valSize];
}
if (valSize){
for (int i=0; i < valSize; i++){
// put my rank in my portion of LHS and my portion of RHS
vals[i] = localProc;
}
}
RHS = new Epetra_MultiVector(Copy, sourceRowMap, vals, 1, 1);
LHS = new Epetra_MultiVector(Copy, *LHSmap, vals, 1, 1);
if (valSize){
delete [] vals;
}
problem = new Epetra_LinearProblem(matrix.get(), LHS, RHS);
Epetra_LinearProblem lp = *problem;
if (lp.CheckInput()){
ERROREXIT((localProc==0), "Error creating a LinearProblem");
}
if (noParams && noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp);
}
else if (noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp, params);
}
targetRowMap = &(problemPtr->GetMatrix()->RowMatrixRowMap());
targetColMap = &(problemPtr->GetMatrix()->RowMatrixColMap());
}
// Redistribute the edge weights
// Comment this out since we don't redistribute columns
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *graphPtr));
targetRowMap = &(newewgts->RowMap());
targetColMap = &(newewgts->ColMap());
}
else{
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *targetRowMap, *targetColMap, 0));
}
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
newewgts->Import(*eptr, *importer, Insert);
newewgts->FillComplete(*targetColMap, *targetRowMap);
costs.setGraphEdgeWeights(newewgts);
}
}
// Redistribute the vertex weights
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetBlockRowMap));
importer = new Epetra_Import(*targetBlockRowMap, sourceRowMap);
}
else{
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetRowMap));
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
}
newvwgts->Import(*vptr, *importer, Insert);
costs.setVertexWeights(newvwgts);
}
if (localProc == 0){
test_type(numPartitions, partitioningType, vertexWeightType, edgeWeightType, objectType);
}
if (verbose){
// Picture of problem before balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("Before load balancing", *problem, Comm);
}
else{
ispatest::show_matrix("Before load balancing", matrix->Graph(), Comm);
}
// Picture of problem after balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("After load balancing (x in Ax=b is not redistributed)", *problemPtr, Comm);
}
else if (objectType == EPETRA_ROWMATRIX){
ispatest::show_matrix("After load balancing", *rowMatrixPtr, Comm);
}
else if (objectType == EPETRA_CRSMATRIX){
ispatest::show_matrix("After load balancing", matrixPtr->Graph(), Comm);
}
else if (objectType == EPETRA_CRSGRAPH){
ispatest::show_matrix("After load balancing", *graphPtr, Comm);
}
}
// After partitioning, recompute the metrics
if (partitioningType == GRAPH_PARTITIONING){
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_graph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_graph_metrics(*rowMatrixPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_graph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else {
rc = ispatest::compute_graph_metrics(*graphPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
}
else{
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_hypergraph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_hypergraph_metrics(*rowMatrixPtr, costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_hypergraph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, cutn2, cutl2);
}
else{
rc = ispatest::compute_hypergraph_metrics(*graphPtr, costs,
myShare, balance2, cutn2, cutl2);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
std::string why;
if (partitioningType == GRAPH_PARTITIONING){
fail = (cutWgt2 > cutWgt1);
why = "New weighted edge cuts are worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << " Total edge cuts: " << numCuts1;
std::cout << " Total weighted edge cuts: " << cutWgt1 << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
std::cout << " Total edge cuts: " << numCuts2;
std::cout << " Total weighted edge cuts: " << cutWgt2 << std::endl;
}
}
else{
fail = (cutl2 > cutl1);
why = "New cutl is worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
}
}
if (fail){
if (localProc == 0) std::cout << "ERROR: "+why << std::endl;
}
// Check that input matrix is valid. This test constructs an "x"
// with the matrix->DomainMap() and a "y" with matrix->RangeMap()
// and then calculates y = Ax.
if (objectType == EPETRA_LINEARPROBLEM){
valid = ispatest::test_matrix_vector_multiply(*problemPtr);
}
else if (objectType == EPETRA_ROWMATRIX){
valid = ispatest::test_row_matrix_vector_multiply(*rowMatrixPtr);
}
else if (objectType == EPETRA_CRSMATRIX){
valid = ispatest::test_matrix_vector_multiply(*matrixPtr);
}
else{
valid = ispatest::test_matrix_vector_multiply(*graphPtr);
}
if (!valid){
if (localProc == 0) std::cout << "Rebalanced matrix is not a valid Epetra matrix" << std::endl;
fail = 1;
}
else{
if (localProc == 0) std::cout << "Rebalanced matrix is a valid Epetra matrix" << std::endl;
}
if (localProc == 0)
std::cout << std::endl;
#else
std::cout << "test_simple main: currently can only test "
<< "with Epetra and EpetraExt enabled." << std::endl;
rc = -1;
#endif
return fail;
}
/* Computes the approximate Schur complement for the wide separator */
Teuchos::RCP<Epetra_CrsMatrix> computeApproxWideSchur(shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
Epetra_CrsMatrix *G, Epetra_CrsMatrix *R,
Epetra_LinearProblem *LP, Amesos_BaseSolver *solver,
Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C,
Epetra_Map *localDRowMap)
{
int i;
double relative_thres = config->relative_threshold;
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
//Epetra_Map CrMap = C->RowMap();
//int *c_rows = CrMap.MyGlobalElements();
//int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
//int c_localElems = CrMap.NumMyElements();
//int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
//Epetra_Map RrMap = R->RowMap();
//int *r_rows = RrMap.MyGlobalElements();
//int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
//int r_localElems = RrMap.NumMyElements();
//int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
//cout << "Created local G matrix" << endl;
int nvectors = 16;
/*ShyLU_Probing_Operator probeop(&localG, &localR, LP, solver, &localC,
localDRowMap, nvectors);*/
ShyLU_Local_Schur_Operator probeop(config, ssym, &localG, R, LP, solver,
ifSolver, C, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[nvectors];
int *indices = new int[nvectors];
double *vecvalues;
#ifdef SHYLU_DEBUG
// mfh 25 May 2015: Don't declare this variable if it's not used.
// It's only used if SHYLU_DEBUG is defined.
int dropped = 0;
#endif // SHYLU_DEBUG
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec (G_localRMap, nvectors);
Epetra_MultiVector Scol (G_localRMap, nvectors);
probevec.PutScalar(0.0);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
// Set the probevec to find block columns of S.
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
// Reset the probevec to all zeros.
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
probevec.ReplaceGlobalValue(g_rows[cindex], k, 0.0);
}
Scol.MaxValue(maxvalue);
nentries = 0;
for (int j = 0 ; j < g_localElems ; j++)
{
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
vecvalues = Scol[k];
if ((g_rows[cindex] == g_rows[j]) ||
(abs(vecvalues[j]/maxvalue[k]) > relative_thres))
// diagonal entry or large entry.
{
values[nentries] = vecvalues[j];
indices[nentries++] = g_rows[cindex];
}
#ifdef SHYLU_DEBUG
else if (vecvalues[j] != 0.0)
{
dropped++;
}
#endif // SHYLU_DEBUG
}
Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
nentries = 0;
}
}
if (i < g_localElems)
{
nvectors = g_localElems - i;
probeop.ResetTempVectors(nvectors);
Epetra_MultiVector probevec1 (G_localRMap, nvectors);
Epetra_MultiVector Scol1 (G_localRMap, nvectors);
probevec1.PutScalar(0.0);
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec1.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec1, Scol1);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol1.MaxValue(maxvalue);
nentries = 0;
for (int j = 0 ; j < g_localElems ; j++)
{
//cout << "MAX" << maxvalue << endl;
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
vecvalues = Scol1[k];
//nentries = 0; // inserting one entry in each row for now
if ((g_rows[cindex] == g_rows[j]) ||
(abs(vecvalues[j]/maxvalue[k]) > relative_thres))
// diagonal entry or large entry.
{
values[nentries] = vecvalues[j];
indices[nentries++] = g_rows[cindex];
}
#ifdef SHYLU_DEBUG
else if (vecvalues[j] != 0.0)
{
dropped++;
}
#endif // SHYLU_DEBUG
}
Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
nentries = 0;
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime()
<< endl;
ftime.reset();
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
probeop.PrintTimingInfo();
#endif
Sbar->FillComplete();
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
#ifdef SHYLU_DEBUG
cout << "#dropped entries" << dropped << endl;
#endif
delete[] values;
delete[] indices;
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
delete[] maxvalue;
return Sbar;
}
int main(int argc, char *argv[]) {
//
int MyPID = 0;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
MyPID = Comm.MyPID();
#else
Epetra_SerialComm Comm;
#endif
//
typedef double ST;
typedef Teuchos::ScalarTraits<ST> SCT;
typedef SCT::magnitudeType MT;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Belos::MultiVecTraits<ST,MV> MVT;
typedef Belos::OperatorTraits<ST,MV,OP> OPT;
using Teuchos::ParameterList;
using Teuchos::RCP;
using Teuchos::rcp;
bool success = false;
bool verbose = false;
try {
bool debug = false, proc_verbose = false;
int frequency = -1; // frequency of status test output.
int numrhs = 1; // number of right-hand sides to solve for
int maxiters = -1; // maximum number of iterations allowed per linear system
MT tol = 1.0e-10; // relative residual tolerance
Teuchos::CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("debug","nodebug",&debug,"Print debugging information from the solver.");
cmdp.setOption("frequency",&frequency,"Solvers frequency for printing residuals (#iters).");
cmdp.setOption("tol",&tol,"Relative residual tolerance used by GMRES solver.");
cmdp.setOption("num-rhs",&numrhs,"Number of right-hand sides to be solved for.");
cmdp.setOption("max-iters",&maxiters,"Maximum number of iterations per linear system (-1 = adapted to problem/block size).");
if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
return -1;
}
if (!verbose)
frequency = -1; // reset frequency if test is not verbose
// **********************************************************************
// ******************Set up the problem to be solved*********************
// construct diagonal matrix
const int NumGlobalElements = 100;
const int m = 4; // number of negative eigenvalues
// Create diagonal matrix with n-m positive and m negative eigenvalues.
Epetra_Map epetraMap( NumGlobalElements, 0, Comm );
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix( Copy, epetraMap, 1 ) );
for ( int k=0; k<epetraMap.NumMyElements(); k++ )
{
int GIDk = epetraMap.GID(k);
double val = 2*(GIDk-m) + 1;
TEUCHOS_ASSERT_EQUALITY( 0, A->InsertGlobalValues( GIDk, 1, &val, &GIDk ) );
}
TEUCHOS_ASSERT_EQUALITY( 0, A->FillComplete() );
TEUCHOS_ASSERT_EQUALITY( 0, A->OptimizeStorage() );
// create initial guess and right-hand side
Teuchos::RCP<Epetra_MultiVector> vecX = Teuchos::rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
Teuchos::RCP<Epetra_MultiVector> vecB = Teuchos::rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
// **********************************************************************
proc_verbose = verbose && (MyPID==0); /* Only print on the zero processor */
Teuchos::RCP<Epetra_MultiVector> X;
Teuchos::RCP<Epetra_MultiVector> B;
// Check to see if the number of right-hand sides is the same as requested.
if (numrhs>1) {
X = rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
B = rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
X->Random();
OPT::Apply( *A, *X, *B );
X->PutScalar( 0.0 );
}
else {
X = Teuchos::rcp_implicit_cast<Epetra_MultiVector>(vecX);
B = Teuchos::rcp_implicit_cast<Epetra_MultiVector>(vecB);
B->PutScalar( 1.0 );
}
//
// ********Other information used by block solver***********
// *****************(can be user specified)******************
//
if (maxiters == -1)
maxiters = NumGlobalElements - 1; // maximum number of iterations to run
//
ParameterList belosList;
belosList.set( "Maximum Iterations", maxiters ); // Maximum number of iterations allowed
belosList.set( "Convergence Tolerance", tol ); // Relative convergence tolerance requested
belosList.set( "Assert Positive Definiteness", false ); // Explicitly don't enforce positive definiteness
int verbosity = Belos::Errors + Belos::Warnings;
if (verbose) {
verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
if (frequency > 0)
belosList.set( "Output Frequency", frequency );
}
if (debug) {
verbosity += Belos::Debug;
}
belosList.set( "Verbosity", verbosity );
//
// Construct an unpreconditioned linear problem instance.
//
Belos::LinearProblem<double,MV,OP> problem( A, X, B );
bool set = problem.setProblem();
if (set == false) {
if (proc_verbose)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return -1;
}
//
// *******************************************************************
// ****************Start the CG iteration*************************
// *******************************************************************
//
// Create an iterative solver manager.
RCP<Belos::SolverManager<double,MV,OP> > newSolver
= rcp( new Belos::PseudoBlockCGSolMgr<double,MV,OP>(rcp(&problem,false), rcp(&belosList,false)));
//
// **********Print out information about problem*******************
//
if (proc_verbose) {
std::cout << std::endl << std::endl;
std::cout << "Dimension of matrix: " << NumGlobalElements << std::endl;
std::cout << "Number of right-hand sides: " << numrhs << std::endl;
std::cout << "Relative residual tolerance: " << tol << std::endl;
std::cout << std::endl;
}
//
// Perform solve
//
Belos::ReturnType ret = newSolver->solve();
//
// Get the number of iterations for this solve.
//
int numIters = newSolver->getNumIters();
if (proc_verbose)
std::cout << "Number of iterations performed for this solve: " << numIters << std::endl;
//
// Compute actual residuals.
//
bool badRes = false;
std::vector<double> actual_resids( numrhs );
std::vector<double> rhs_norm( numrhs );
Epetra_MultiVector resid(epetraMap, numrhs);
OPT::Apply( *A, *X, resid );
MVT::MvAddMv( -1.0, resid, 1.0, *B, resid );
MVT::MvNorm( resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
if (proc_verbose) {
std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i=0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
std::cout<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
if (actRes > tol) badRes = true;
}
}
success = ret==Belos::Converged && !badRes;
if (success) {
if (proc_verbose)
std::cout << "End Result: TEST PASSED" << std::endl;
} else {
if (proc_verbose)
std::cout << "End Result: TEST FAILED" << std::endl;
}
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[]) {
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool testFailed;
bool boolret;
int MyPID = Comm.MyPID();
bool verbose = true;
bool debug = false;
std::string which("SM");
Teuchos::CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("debug","nodebug",&debug,"Print debugging information.");
cmdp.setOption("sort",&which,"Targetted eigenvalues (SM,LM,SR,LR,SI,or LI).");
if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return -1;
}
typedef double ScalarType;
typedef Teuchos::ScalarTraits<ScalarType> ScalarTypeTraits;
typedef ScalarTypeTraits::magnitudeType MagnitudeType;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<ScalarType,MV> MVTraits;
typedef Anasazi::OperatorTraits<ScalarType,MV,OP> OpTraits;
// Dimension of the matrix
int nx = 10; // Discretization points in any one direction.
int NumGlobalElements = nx*nx; // Size of matrix nx*nx
// Construct a Map that puts approximately the same number of
// equations on each processor.
Epetra_Map Map(NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map.MyGlobalElements(&MyGlobalElements[0]);
// Create an integer vector NumNz that is used to build the Petra Matrix.
// NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation
// on this processor
std::vector<int> NumNz(NumMyElements);
/* We are building a matrix of block structure:
| T -I |
|-I T -I |
| -I T |
| ... -I|
| -I T|
where each block is dimension nx by nx and the matrix is on the order of
nx*nx. The block T is a tridiagonal matrix.
*/
for (int i=0; i<NumMyElements; i++) {
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1 ||
MyGlobalElements[i] == nx-1 || MyGlobalElements[i] == nx*(nx-1) ) {
NumNz[i] = 3;
}
else if (MyGlobalElements[i] < nx || MyGlobalElements[i] > nx*(nx-1) ||
MyGlobalElements[i]%nx == 0 || (MyGlobalElements[i]+1)%nx == 0) {
NumNz[i] = 4;
}
else {
NumNz[i] = 5;
}
}
// Create an Epetra_Matrix
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix(Copy, Map, &NumNz[0]) );
// Diffusion coefficient, can be set by user.
// When rho*h/2 <= 1, the discrete convection-diffusion operator has real eigenvalues.
// When rho*h/2 > 1, the operator has complex eigenvalues.
double rho = 2*(nx+1);
// Compute coefficients for discrete convection-diffution operator
const double one = 1.0;
std::vector<double> Values(4);
std::vector<int> Indices(4);
double h = one /(nx+1);
double h2 = h*h;
double c = 5.0e-01*rho/ h;
Values[0] = -one/h2 - c; Values[1] = -one/h2 + c; Values[2] = -one/h2; Values[3]= -one/h2;
double diag = 4.0 / h2;
int NumEntries, info;
for (int i=0; i<NumMyElements; i++)
{
if (MyGlobalElements[i]==0)
{
Indices[0] = 1;
Indices[1] = nx;
NumEntries = 2;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
assert( info==0 );
}
else if (MyGlobalElements[i] == nx*(nx-1))
{
Indices[0] = nx*(nx-1)+1;
Indices[1] = nx*(nx-2);
NumEntries = 2;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
assert( info==0 );
}
else if (MyGlobalElements[i] == nx-1)
{
Indices[0] = nx-2;
NumEntries = 1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert( info==0 );
Indices[0] = 2*nx-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
assert( info==0 );
}
else if (MyGlobalElements[i] == NumGlobalElements-1)
{
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert( info==0 );
Indices[0] = nx*(nx-1)-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
assert( info==0 );
}
else if (MyGlobalElements[i] < nx)
{
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert( info==0 );
}
else if (MyGlobalElements[i] > nx*(nx-1))
{
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
NumEntries = 3;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert( info==0 );
}
else if (MyGlobalElements[i]%nx == 0)
{
Indices[0] = MyGlobalElements[i]+1;
Indices[1] = MyGlobalElements[i]-nx;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
assert( info==0 );
}
else if ((MyGlobalElements[i]+1)%nx == 0)
{
Indices[0] = MyGlobalElements[i]-nx;
Indices[1] = MyGlobalElements[i]+nx;
NumEntries = 2;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
assert( info==0 );
Indices[0] = MyGlobalElements[i]-1;
NumEntries = 1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert( info==0 );
}
else
{
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
Indices[3] = MyGlobalElements[i]+nx;
NumEntries = 4;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert( info==0 );
}
// Put in the diagonal entry
info = A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, &MyGlobalElements[i]);
assert( info==0 );
}
// Finish up
info = A->FillComplete();
assert( info==0 );
A->SetTracebackMode(1); // Shutdown Epetra Warning tracebacks
//************************************
// Start the block Davidson iteration
//***********************************
//
// Variables used for the Generalized Davidson Method
//
int nev = 4;
int blockSize = 1;
int maxDim = 50;
int restartDim = 10;
int maxRestarts = 500;
double tol = 1e-10;
// Set verbosity level
int verbosity = Anasazi::Errors + Anasazi::Warnings;
if (verbose) {
verbosity += Anasazi::FinalSummary + Anasazi::TimingDetails;
}
if (debug) {
verbosity += Anasazi::Debug;
}
//
// Create parameter list to pass into solver manager
//
Teuchos::ParameterList MyPL;
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Which", which );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Maximum Subspace Dimension", maxDim);
MyPL.set( "Restart Dimension", restartDim);
MyPL.set( "Maximum Restarts", maxRestarts );
MyPL.set( "Convergence Tolerance", tol );
MyPL.set( "Relative Convergence Tolerance", true );
MyPL.set( "Initial Guess", "User" );
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(Map, blockSize) );
ivec->Random();
// Create the eigenproblem.
Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > MyProblem = Teuchos::rcp(
new Anasazi::BasicEigenproblem<double,MV,OP>() );
MyProblem->setA(A);
MyProblem->setInitVec(ivec);
// Inform the eigenproblem that the operator A is non-Hermitian
MyProblem->setHermitian(false);
// Set the number of eigenvalues requested
MyProblem->setNEV( nev );
// Inform the eigenproblem that you are finishing passing it information
boolret = MyProblem->setProblem();
if (boolret != true) {
if (verbose && MyPID == 0) {
std::cout << "Anasazi::BasicEigenproblem::setProblem() returned with error." << std::endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
// Initialize the Block Arnoldi solver
Anasazi::GeneralizedDavidsonSolMgr<double, MV, OP> MySolverMgr(MyProblem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMgr.solve();
testFailed = false;
if (returnCode != Anasazi::Converged && MyPID==0 && verbose) {
testFailed = true;
}
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<ScalarType,MV> sol = MyProblem->getSolution();
std::vector<Anasazi::Value<ScalarType> > evals = sol.Evals;
Teuchos::RCP<MV> evecs = sol.Evecs;
std::vector<int> index = sol.index;
int numev = sol.numVecs;
// Output computed eigenvalues and their direct residuals
if (verbose && MyPID==0) {
int numritz = (int)evals.size();
std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
std::cout<<std::endl<< "Computed Ritz Values"<< std::endl;
std::cout<< std::setw(16) << "Real Part"
<< std::setw(16) << "Imag Part"
<< std::endl;
std::cout<<"-----------------------------------------------------------"<<std::endl;
for (int i=0; i<numritz; i++) {
std::cout<< std::setw(16) << evals[i].realpart
<< std::setw(16) << evals[i].imagpart
<< std::endl;
}
std::cout<<"-----------------------------------------------------------"<<std::endl;
}
if (numev > 0) {
// Compute residuals.
Teuchos::LAPACK<int,double> lapack;
std::vector<double> normA(numev);
// The problem is non-Hermitian.
int i=0;
std::vector<int> curind(1);
std::vector<double> resnorm(1), tempnrm(1);
Teuchos::RCP<MV> tempAevec;
Teuchos::RCP<const MV> evecr, eveci;
Epetra_MultiVector Aevec(Map,numev);
// Compute A*evecs
OpTraits::Apply( *A, *evecs, Aevec );
Teuchos::SerialDenseMatrix<int,double> Breal(1,1), Bimag(1,1);
while (i<numev) {
if (index[i]==0) {
// Get a view of the current eigenvector (evecr)
curind[0] = i;
evecr = MVTraits::CloneView( *evecs, curind );
// Get a copy of A*evecr
tempAevec = MVTraits::CloneCopy( Aevec, curind );
// Compute A*evecr - lambda*evecr
Breal(0,0) = evals[i].realpart;
MVTraits::MvTimesMatAddMv( -1.0, *evecr, Breal, 1.0, *tempAevec );
// Compute the norm of the residual and increment counter
MVTraits::MvNorm( *tempAevec, resnorm );
normA[i] = resnorm[0] / Teuchos::ScalarTraits<MagnitudeType>::magnitude( evals[i].realpart );
i++;
} else {
// Get a view of the real part of the eigenvector (evecr)
curind[0] = i;
evecr = MVTraits::CloneView( *evecs, curind );
// Get a copy of A*evecr
tempAevec = MVTraits::CloneCopy( Aevec, curind );
// Get a view of the imaginary part of the eigenvector (eveci)
curind[0] = i+1;
eveci = MVTraits::CloneView( *evecs, curind );
// Set the eigenvalue into Breal and Bimag
Breal(0,0) = evals[i].realpart;
Bimag(0,0) = evals[i].imagpart;
// Compute A*evecr - evecr*lambdar + eveci*lambdai
MVTraits::MvTimesMatAddMv( -1.0, *evecr, Breal, 1.0, *tempAevec );
MVTraits::MvTimesMatAddMv( 1.0, *eveci, Bimag, 1.0, *tempAevec );
MVTraits::MvNorm( *tempAevec, tempnrm );
// Get a copy of A*eveci
tempAevec = MVTraits::CloneCopy( Aevec, curind );
// Compute A*eveci - eveci*lambdar - evecr*lambdai
MVTraits::MvTimesMatAddMv( -1.0, *evecr, Bimag, 1.0, *tempAevec );
MVTraits::MvTimesMatAddMv( -1.0, *eveci, Breal, 1.0, *tempAevec );
MVTraits::MvNorm( *tempAevec, resnorm );
// Compute the norms and scale by magnitude of eigenvalue
normA[i] = lapack.LAPY2( tempnrm[0], resnorm[0] ) /
lapack.LAPY2( evals[i].realpart, evals[i].imagpart );
normA[i+1] = normA[i];
i=i+2;
}
}
// Output computed eigenvalues and their direct residuals
if (verbose && MyPID==0) {
std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
std::cout<<std::endl<< "Actual Residuals"<<std::endl;
std::cout<< std::setw(16) << "Real Part"
<< std::setw(16) << "Imag Part"
<< std::setw(20) << "Direct Residual"<< std::endl;
std::cout<<"-----------------------------------------------------------"<<std::endl;
for (int j=0; j<numev; j++) {
std::cout<< std::setw(16) << evals[j].realpart
<< std::setw(16) << evals[j].imagpart
<< std::setw(20) << normA[j] << std::endl;
if ( normA[j] > tol ) {
testFailed = true;
}
}
std::cout<<"-----------------------------------------------------------"<<std::endl;
}
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
if (testFailed) {
if (verbose && MyPID==0) {
std::cout << "End Result: TEST FAILED" << std::endl;
}
return -1;
}
//
// Default return value
//
if (verbose && MyPID==0) {
std::cout << "End Result: TEST PASSED" << std::endl;
}
return 0;
}
void
twoD_diffusion_ME::
evalModel(const InArgs& inArgs, const OutArgs& outArgs) const
{
//
// Determinisic calculation
//
// Solution vector
Teuchos::RCP<const Epetra_Vector> det_x = inArgs.get_x();
// Parameters
Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(0);
if (p == Teuchos::null)
p = p_init;
Teuchos::RCP<Epetra_Vector> f = outArgs.get_f();
Teuchos::RCP<Epetra_Operator> W = outArgs.get_W();
Teuchos::RCP<Epetra_Operator> WPrec = outArgs.get_WPrec();
if (f != Teuchos::null || W != Teuchos::null || WPrec != Teuchos::null) {
if (basis != Teuchos::null) {
for (int i=0; i<point.size(); i++)
point[i] = (*p)[i];
basis->evaluateBases(point, basis_vals);
A->PutScalar(0.0);
for (int k=0;k<A_k.size();k++)
EpetraExt::MatrixMatrix::Add((*A_k[k]), false, basis_vals[k], *A, 1.0);
}
else {
*A = *(A_k[0]);
for (int k=1;k<A_k.size();k++)
EpetraExt::MatrixMatrix::Add((*A_k[k]), false, (*p)[k-1], *A, 1.0);
}
A->FillComplete();
A->OptimizeStorage();
}
// Residual
if (f != Teuchos::null) {
Teuchos::RCP<Epetra_Vector> kx = Teuchos::rcp(new Epetra_Vector(*x_map));
A->Apply(*det_x,*kx);
f->Update(1.0,*kx,-1.0, *b, 0.0);
}
// Jacobian
if (W != Teuchos::null) {
Teuchos::RCP<Epetra_CrsMatrix> jac =
Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W, true);
*jac = *A;
jac->FillComplete();
jac->OptimizeStorage();
}
// Preconditioner
if (WPrec != Teuchos::null)
precFactory->recompute(A, WPrec);
// Responses (mean value)
Teuchos::RCP<Epetra_Vector> g = outArgs.get_g(0);
if (g != Teuchos::null) {
(det_x->MeanValue(&(*g)[0]));
(*g)[0] *= double(det_x->GlobalLength()) / double(mesh.size());
}
//
// Stochastic Galerkin calculation
//
// Stochastic solution vector
InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();
// Stochastic parameters
InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(0);
// Stochastic residual
OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
if (f_sg != Teuchos::null) {
// Get stochastic expansion data
Teuchos::RCP<Stokhos::OrthogPolyExpansion<int,double> > expn =
inArgs.get_sg_expansion();
typedef Stokhos::Sparse3Tensor<int,double> Cijk_type;
Teuchos::RCP<const Cijk_type> Cijk = expn->getTripleProduct();
const Teuchos::Array<double>& norms = basis->norm_squared();
if (sg_kx_vec_all.size() != basis->size()) {
sg_kx_vec_all.resize(basis->size());
for (int i=0;i<basis->size();i++) {
sg_kx_vec_all[i] = Teuchos::rcp(new Epetra_Vector(*x_map));
}
}
f_sg->init(0.0);
Cijk_type::k_iterator k_begin = Cijk->k_begin();
Cijk_type::k_iterator k_end = Cijk->k_end();
for (Cijk_type::k_iterator k_it=k_begin; k_it!=k_end; ++k_it) {
int k = Stokhos::index(k_it);
for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
j_it != Cijk->j_end(k_it); ++j_it) {
int j = Stokhos::index(j_it);
A_k[k]->Apply((*x_sg)[j],*(sg_kx_vec_all[j]));
}
for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
j_it != Cijk->j_end(k_it); ++j_it) {
int j = Stokhos::index(j_it);
for (Cijk_type::kji_iterator i_it = Cijk->i_begin(j_it);
i_it != Cijk->i_end(j_it); ++i_it) {
int i = Stokhos::index(i_it);
double c = Stokhos::value(i_it); // C(i,j,k)
(*f_sg)[i].Update(1.0*c/norms[i],*(sg_kx_vec_all[j]),1.0);
}
}
} //End
(*f_sg)[0].Update(-1.0,*b,1.0);
}
// Stochastic Jacobian
OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
if (W_sg != Teuchos::null) {
for (int i=0; i<W_sg->size(); i++) {
Teuchos::RCP<Epetra_CrsMatrix> jac =
Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i), true);
*jac = *A_k[i];
jac->FillComplete();
jac->OptimizeStorage();
}
}
// Stochastic responses
Teuchos::RCP< Stokhos::EpetraVectorOrthogPoly > g_sg =
outArgs.get_g_sg(0);
if (g_sg != Teuchos::null) {
int sz = x_sg->size();
for (int i=0; i<sz; i++) {
(*x_sg)[i].MeanValue(&(*g_sg)[i][0]);
(*g_sg)[i][0] *= double((*x_sg)[i].GlobalLength()) / double(mesh.size());
}
}
//
// Multi-point calculation
//
// Stochastic solution vector
mp_const_vector_t x_mp = inArgs.get_x_mp();
// Stochastic parameters
mp_const_vector_t p_mp = inArgs.get_p_mp(0);
// Stochastic residual
mp_vector_t f_mp = outArgs.get_f_mp();
mp_operator_t W_mp = outArgs.get_W_mp();
if (f_mp != Teuchos::null || W_mp != Teuchos::null) {
int num_mp = x_mp->size();
for (int i=0; i<num_mp; i++) {
// Compute operator
if (basis != Teuchos::null) {
for (int k=0; k<point.size(); k++)
point[k] = (*p_mp)[i][k];
basis->evaluateBases(point, basis_vals);
A->PutScalar(0.0);
for (int k=0;k<A_k.size();k++)
EpetraExt::MatrixMatrix::Add((*A_k[k]), false, basis_vals[k], *A,
1.0);
}
else {
*A = *(A_k[0]);
for (int k=1;k<A_k.size();k++)
EpetraExt::MatrixMatrix::Add((*A_k[k]), false, (*p_mp)[i][k-1], *A,
1.0);
}
A->FillComplete();
A->OptimizeStorage();
// Compute residual
if (f_mp != Teuchos::null) {
A->Apply((*x_mp)[i], (*f_mp)[i]);
(*f_mp)[i].Update(-1.0, *b, 1.0);
}
// Copy operator
if (W_mp != Teuchos::null) {
Teuchos::RCP<Epetra_CrsMatrix> jac =
Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i),
true);
*jac = *A;
jac->FillComplete();
jac->OptimizeStorage();
}
}
}
// Multipoint responses
mp_vector_t g_mp = outArgs.get_g_mp(0);
if (g_mp != Teuchos::null) {
int sz = x_mp->size();
for (int i=0; i<sz; i++) {
(*x_mp)[i].MeanValue(&(*g_mp)[i][0]);
(*g_mp)[i][0] *= double((*x_mp)[i].GlobalLength()) / double(mesh.size());
}
}
}
void
createEpetraProblem (const Teuchos::RCP<const Epetra_Comm>& epetraComm,
const std::string& filename,
Teuchos::RCP<Epetra_Map>& rowMap,
Teuchos::RCP<Epetra_CrsMatrix>& A,
Teuchos::RCP<Epetra_MultiVector>& B,
Teuchos::RCP<Epetra_MultiVector>& X,
int& numRHS)
{
using Teuchos::inOutArg;
using Teuchos::ptr;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::set_extra_data;
const int MyPID = epetraComm->MyPID();
int n_nonzeros, N_update;
int *bindx = NULL, *update = NULL, *col_inds = NULL;
double *val = NULL, *row_vals = NULL;
double *xguess = NULL, *b = NULL, *xexact = NULL;
//
// Set up the problem to be solved
//
int NumGlobalElements; // total # of rows in matrix
try {
// Read in matrix from HB file
Trilinos_Util_read_hb (const_cast<char *> (filename.c_str()), MyPID,
&NumGlobalElements, &n_nonzeros, &val, &bindx,
&xguess, &b, &xexact);
// Distribute data among processors
Trilinos_Util_distrib_msr_matrix (*epetraComm, &NumGlobalElements,
&n_nonzeros, &N_update, &update, &val,
&bindx, &xguess, &b, &xexact);
//
// Construct the matrix
//
int NumMyElements = N_update; // # local rows of matrix on processor
//
// Create an int array NumNz that is used to build the Petra Matrix.
// NumNz[i] is the number of OFF-DIAGONAL terms for the i-th global
// equation on this processor.
//
std::vector<int> NumNz (NumMyElements);
for (int i = 0; i < NumMyElements; ++i) {
NumNz[i] = bindx[i+1] - bindx[i] + 1;
}
rowMap = rcp (new Epetra_Map (NumGlobalElements, NumMyElements,
update, 0, *epetraComm));
//set_extra_data (epetraComm, "Map::Comm", inOutArg (rowMap));
// Create an Epetra sparse matrix.
if (NumMyElements == 0) {
A = rcp (new Epetra_CrsMatrix (Copy, *rowMap, static_cast<int*>(NULL)));
} else {
A = rcp (new Epetra_CrsMatrix (Copy, *rowMap, &NumNz[0]));
}
//set_extra_data (rowMap, "Operator::Map", ptr (A));
// Add rows to the sparse matrix one at a time.
int NumEntries;
for (int i = 0; i < NumMyElements; ++i) {
row_vals = val + bindx[i];
col_inds = bindx + bindx[i];
NumEntries = bindx[i+1] - bindx[i];
int info = A->InsertGlobalValues (update[i], NumEntries, row_vals, col_inds);
TEUCHOS_TEST_FOR_EXCEPTION( info != 0, std::logic_error,
"Failed to insert global value into A." );
info = A->InsertGlobalValues (update[i], 1, val+i, update+i);
TEUCHOS_TEST_FOR_EXCEPTION( info != 0, std::logic_error,
"Failed to insert global value into A." );
}
// Finish initializing the sparse matrix.
int info = A->FillComplete();
TEUCHOS_TEST_FOR_EXCEPTION( info != 0, std::logic_error,
"FillComplete() failed on the sparse matrix A.");
info = A->OptimizeStorage();
TEUCHOS_TEST_FOR_EXCEPTION( info != 0, std::logic_error,
"OptimizeStorage() failed on the sparse matrix A.");
A->SetTracebackMode (1); // Shut down Epetra warning tracebacks
//
// Construct the right-hand side and solution multivectors.
//
if (false && b != NULL) {
B = rcp (new Epetra_MultiVector (::Copy, *rowMap, b, NumMyElements, 1));
numRHS = 1;
} else {
B = rcp (new Epetra_MultiVector (*rowMap, numRHS));
B->Random ();
}
X = rcp (new Epetra_MultiVector (*rowMap, numRHS));
X->PutScalar (0.0);
//set_extra_data (rowMap, "X::Map", Teuchos::ptr (X));
//set_extra_data (rowMap, "B::Map", Teuchos::ptr (B));
}
catch (std::exception& e) {
// Free up memory before rethrowing the exception.
if (update) free(update);
if (val) free(val);
if (bindx) free(bindx);
if (xexact) free(xexact);
if (xguess) free(xguess);
if (b) free(b);
throw e; // Rethrow the exception.
}
catch (...) { // Epetra sometimes throws an int "object."
// Free up memory before rethrowing the exception.
if (update) free(update);
if (val) free(val);
if (bindx) free(bindx);
if (xexact) free(xexact);
if (xguess) free(xguess);
if (b) free(b);
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
"Generating the Epetra problem to solve failed "
"with an unknown error.");
}
//if (false) {
//
// Create workspace
//
//Teuchos::set_default_workspace_store (rcp (new Teuchos::WorkspaceStoreInitializeable(static_cast<size_t>(2e+6))));
//}
//
// Free up memory.
//
if (update) free(update);
if (val) free(val);
if (bindx) free(bindx);
if (xexact) free(xexact);
if (xguess) free(xguess);
if (b) free(b);
}