RCP<Epetra_CrsMatrix> getMyEpetraMatrix(int numRows, int numCols, double shift=0.0)
{
const RCP<const Epetra_Comm> comm = getEpetraComm();
const Epetra_Map rowMap(numRows, 0, *comm);
const Epetra_Map domainMap(numCols, numCols, 0, *comm);
const RCP<Epetra_CrsMatrix> epetraCrsM =
rcp(new Epetra_CrsMatrix(Copy, rowMap,domainMap,0));
Array<double> rowEntries(numCols);
Array<int> columnIndices(numCols);
for (int j = 0; j < numCols; ++j)
columnIndices[j] = j;
const int numLocalRows = rowMap.NumMyElements();
for (int i = 0; i < numLocalRows; ++i) {
for (int j = 0; j < numCols; ++j) {
rowEntries[j] = as<double>(i+1) + as<double>(j+1) / 10 + shift;
}
epetraCrsM->InsertMyValues( i, numCols, &rowEntries[0], &columnIndices[0] );
}
epetraCrsM->FillComplete(domainMap,rowMap);
return epetraCrsM;
}
// =========================================================================
RCP<Epetra_CrsMatrix>
contructEpetraMatrix(const int n,
const Epetra_Comm & eComm)
{
RCP<Epetra_CrsMatrix> A;
// Build the matrix (-1,2,-1).
Epetra_Map map(n, 0, eComm);
int * myGlobalElements = map.MyGlobalElements();
A = rcp(new Epetra_CrsMatrix(Copy, map, 3));
double vals[] = {-1.0, 2.0, -1.0};
for (int k = 0; k < map.NumMyElements(); k++) {
if (myGlobalElements[k] == 0) {
int cols[] = {myGlobalElements[k], myGlobalElements[k]+1};
TEUCHOS_ASSERT_EQUALITY(0, A->InsertGlobalValues(myGlobalElements[k],
2,
&vals[1],
cols));
} else if (myGlobalElements[k] == n-1) {
int cols[] = {myGlobalElements[k]-1, myGlobalElements[k]};
TEUCHOS_ASSERT_EQUALITY(0, A->InsertGlobalValues(myGlobalElements[k],
2,
vals,
cols));
} else {
int cols[] = {myGlobalElements[k]-1, myGlobalElements[k], myGlobalElements[k]+1};
TEUCHOS_ASSERT_EQUALITY(0, A->InsertGlobalValues(myGlobalElements[k],
3,
vals,
cols));
}
}
TEUCHOS_ASSERT_EQUALITY(0, A->FillComplete(true));
return A;
}
int rebalanceEpetraProblem( RCP<Epetra_Map> &Map,
RCP<Epetra_CrsMatrix> &A,
RCP<Epetra_MultiVector> &B,
RCP<Epetra_MultiVector> &X,
Epetra_Comm &Comm
)
{
// Rebalance linear system across multiple processors.
if ( Comm.NumProc() > 1 ) {
RCP<Epetra_Map> newMap = rcp( new Epetra_Map( Map->NumGlobalElements(), Map->IndexBase(), Comm ) );
RCP<Epetra_Import> newImport = rcp( new Epetra_Import( *newMap, *Map ) );
// Create rebalanced versions of the linear system.
RCP<Epetra_CrsMatrix> newA = rcp( new Epetra_CrsMatrix( BELOSEPETRACOPY, *newMap, 0 ) );
newA->Import( *A, *newImport, Insert );
newA->FillComplete();
RCP<Epetra_MultiVector> newB = rcp( new Epetra_MultiVector( *newMap, B->NumVectors() ) );
newB->Import( *B, *newImport, Insert );
RCP<Epetra_MultiVector> newX = rcp( new Epetra_MultiVector( *newMap, X->NumVectors() ) );
newX->Import( *X, *newImport, Insert );
// Set the pointers to the new rebalance linear system.
A = newA;
B = newB;
X = newX;
Map = newMap;
}
return (0);
}
RCP<Epetra_CrsMatrix> UserInputForTests::getEpetraCrsMatrix()
{
if (M_.is_null())
throw std::runtime_error("could not read mtx file");
RCP<Epetra_CrsGraph> egraph = getEpetraCrsGraph();
eM_ = rcp(new Epetra_CrsMatrix(Copy, *egraph));
size_t maxRow = M_->getNodeMaxNumRowEntries();
int nrows = egraph->NumMyRows();
int base = egraph->IndexBase();
const Epetra_BlockMap &rowMap = egraph->RowMap();
const Epetra_BlockMap &colMap = egraph->ColMap();
Array<int> colGid(maxRow);
for (int i=0; i < nrows; i++){
ArrayView<const int> colLid;
ArrayView<const scalar_t> nz;
M_->getLocalRowView(i+base, colLid, nz);
size_t rowSize = colLid.size();
int rowGid = rowMap.GID(i+base);
for (size_t j=0; j < rowSize; j++){
colGid[j] = colMap.GID(colLid[j]);
}
eM_->InsertGlobalValues(
rowGid, rowSize, nz.getRawPtr(), colGid.getRawPtr());
}
eM_->FillComplete();
return eM_;
}
RCP<Epetra_Operator>
MockModelEval_D::
create_W() const
{
RCP<Epetra_CrsMatrix> W = rcp(new Epetra_CrsMatrix(Copy, *graph));
W->FillComplete();
return W;
}
// construct a diagonal matrix
const Teuchos::RCP<const Thyra::LinearOpBase<double> > DiagMatrix(int cnt,double * vec,std::string label)
{
const RCP<Epetra_SerialComm> comm = rcp(new Epetra_SerialComm());
const RCP<Epetra_Map> map = rcp(new Epetra_Map(cnt,0,*comm));
const RCP<Epetra_CrsMatrix> ptrF = rcp(new Epetra_CrsMatrix(Copy,*map,1));
// construct a diagonal matrix
for(int i=0;i<cnt;i++)
ptrF->InsertGlobalValues(i,1,&vec[i],&i);
ptrF->FillComplete();
// return thyra object
return Thyra::epetraLinearOp(ptrF,label);
}
const RCP<Epetra_Operator> buildStridedSystem(const Epetra_Comm & comm,int size)
{
Epetra_Map map(2*size,0,comm);
RCP<Epetra_CrsMatrix> mat = rcp(new Epetra_CrsMatrix(Copy,map,0));
int numUnks = 2;
double valuesA[] = { -1.0, 2.0, 7.0, -1.0 };
double valuesB[] = { -1.0, 2.0, 9.0, -1.0 };
int iTempA[] = {-numUnks,0, 1,numUnks}, indices[4];
int iTempB[] = {-numUnks,0,-1,numUnks};
double * vPtr;
int * iPtr;
for(int i=0;i<map.NumMyElements()/numUnks;i++) {
int count = 4;
int gidA = map.GID(2*i);
int gidB = gidA+1;
for(int n=0;n<numUnks;n++) {
int * iTemp = (n==0) ? iTempA : iTempB;
int gid = (n==0) ? gidA : gidB;
indices[0] = gid+iTemp[0];
indices[1] = gid+iTemp[1];
indices[2] = gid+iTemp[2];
indices[3] = gid+iTemp[3];
vPtr = (n==0) ? valuesA : valuesB;
iPtr = indices;
if(gid<numUnks) {
vPtr++;
iPtr = &indices[1];
count = 3;
}
else if(gid>map.MaxAllGID()-numUnks)
count = 3;
mat->InsertGlobalValues(gid,count,vPtr,iPtr);
}
}
mat->FillComplete();
EpetraExt::RowMatrixToMatrixMarketFile("strided.mm",*mat);
return mat;
}
RCP<Thyra::LinearOpBase<double> > buildExampleOp(int type,const Epetra_Comm & comm,double scale)
{
Epetra_Map map(3,0,comm);
RCP<Epetra_CrsMatrix> mat = rcp(new Epetra_CrsMatrix(Copy,map,3));
double values[3];
int indices[3] = { 0, 1, 2 };
switch(type) {
case 3:
values[0] = 9.0; values[1] = 8.0; values[2] = 7.0;
mat->InsertGlobalValues(0,3,values,indices);
values[0] = 1.0; values[1] = 2.0; values[2] = 3.0;
mat->InsertGlobalValues(1,3,values,indices);
values[0] = 4.0; values[1] = 5.0; values[2] = 6.0;
mat->InsertGlobalValues(2,3,values,indices);
break;
case 2:
values[0] = 5.0; values[1] = 0.0; values[2] = 6.0;
mat->InsertGlobalValues(0,3,values,indices);
values[0] = 2.0; values[1] = 6.0; values[2] = 0.0;
mat->InsertGlobalValues(1,3,values,indices);
values[0] = 0.0; values[1] = 3.0; values[2] = 7.0;
mat->InsertGlobalValues(2,3,values,indices);
break;
case 1:
values[0] = 1.0; values[1] = 0.0; values[2] = 2.0;
mat->InsertGlobalValues(0,3,values,indices);
values[0] = 2.0; values[1] = 5.0; values[2] = 0.0;
mat->InsertGlobalValues(1,3,values,indices);
values[0] = 0.0; values[1] = 3.0; values[2] = 1.0;
mat->InsertGlobalValues(2,3,values,indices);
default:
break;
}
mat->FillComplete();
mat->Scale(scale);
return Thyra::nonconstEpetraLinearOp(mat);
}
static const RCP<const Thyra::LinearOpBase<double> > build2x2(const RCP<const Epetra_Map> & map,double a,double b,double c,double d)
{
int indicies[2];
double row0[2];
double row1[2];
indicies[0] = 0;
indicies[1] = 1;
// build a CrsMatrix
RCP<Epetra_CrsMatrix> blk = rcp(new Epetra_CrsMatrix(Copy,*map,2));
row0[0] = a; row0[1] = c; // do a transpose here!
row1[0] = b; row1[1] = d;
blk->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
blk->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
blk->FillComplete();
return Thyra::epetraLinearOp(blk);
}
void newAssembly(Teuchos::FancyOStream &out)
{
RCP<panzer::DOFManager<int,int> > my_dofM;
RCP<Map> rowmap;
RCP<Map> colmap;
RCP<panzer::ConnManager<int,int> > conn;
const std::vector<int> & myElements=conn->getElementBlock("eblock-0_0_0");
std::vector<std::vector<int> > gids;
std::vector<std::vector<int> > lids;
std::vector< std::vector<std::vector<double> > >miniMat;
fillMeUp1(gids,lids,miniMat,my_dofM,myElements,colmap);
RCP<CrsGraph> crsgraph = rcp(new CrsGraph(Copy,*rowmap,*colmap,-1));
//Tell the graph where elements will be.
for(size_t e=0;e<myElements.size();++e){
for (size_t i = 0; i < gids[e].size(); ++i) {
crsgraph->InsertGlobalIndices(gids[e][i],gids[e].size(), &gids[e][0]);
}
}
{
crsgraph->FillComplete();
}
RCP<CrsMatrix> crsmat = rcp(new CrsMatrix(Copy,*crsgraph));
//Where the data transfer takes place.
for(std::size_t i=0;i<20;i++) {
Teuchos::TimeMonitor LocalTimer(*New_Time);
for ( size_t e = 0; e < myElements.size(); ++e) {
for (size_t i = 0; i < gids[e].size(); ++i) {
int accid=lids[e][i];
crsmat->SumIntoMyValues(accid,lids[e].size(),&miniMat[e][i][0],&lids[e][0]);
}
}
}
return;
}
const RCP<Thyra::LinearOpBase<double> > buildSystem(const Epetra_Comm & comm,int size)
{
Epetra_Map map(size,0,comm);
RCP<Epetra_CrsMatrix> mat = rcp(new Epetra_CrsMatrix(Copy,map,0));
double values[] = { -1.0, 2.0, -1.0};
int iTemp[] = {-1,0,1}, indices[3];
double * vPtr;
int * iPtr;
for(int i=0;i<map.NumMyElements();i++) {
int count = 3;
int gid = map.GID(i);
vPtr = values;
iPtr = indices;
indices[0] = gid+iTemp[0];
indices[1] = gid+iTemp[1];
indices[2] = gid+iTemp[2];
if(gid==0) {
vPtr = &values[1];
iPtr = &indices[1];
count = 2;
}
else if(gid==map.MaxAllGID())
count = 2;
mat->InsertGlobalValues(gid,count,vPtr,iPtr);
}
mat->FillComplete();
return Thyra::nonconstEpetraLinearOp(mat);
}
RCP<Epetra_CrsGraph> UserInputForTests::getEpetraCrsGraph()
{
if (M_.is_null())
throw std::runtime_error("could not read mtx file");
RCP<const tcrsGraph_t> tgraph = M_->getCrsGraph();
RCP<const Tpetra::Map<lno_t, gno_t> > trowMap = tgraph->getRowMap();
RCP<const Tpetra::Map<lno_t, gno_t> > tcolMap = tgraph->getColMap();
int nElts = static_cast<int>(trowMap->getGlobalNumElements());
int nMyElts = static_cast<int>(trowMap->getNodeNumElements());
int base = trowMap->getIndexBase();
ArrayView<const int> gids = trowMap->getNodeElementList();
Epetra_BlockMap erowMap(nElts, nMyElts,
gids.getRawPtr(), 1, base, *ecomm_);
Array<int> rowSize(nMyElts);
for (int i=0; i < nMyElts; i++){
rowSize[i] = static_cast<int>(M_->getNumEntriesInLocalRow(i+base));
}
size_t maxRow = M_->getNodeMaxNumRowEntries();
Array<int> colGids(maxRow);
ArrayView<const int> colLid;
eG_ = rcp(new Epetra_CrsGraph(Copy, erowMap,
rowSize.getRawPtr(), true));
for (int i=0; i < nMyElts; i++){
tgraph->getLocalRowView(i+base, colLid);
for (int j=0; j < colLid.size(); j++)
colGids[j] = tcolMap->getGlobalElement(colLid[j]);
eG_->InsertGlobalIndices(gids[i], rowSize[i], colGids.getRawPtr());
}
eG_->FillComplete();
return eG_;
}
// Serial only. Very straightforward: one eval per column. Could speed up a lot
// using graph coloring.
double
testJacobian (panzer::AssemblyEngine_TemplateManager<panzer::Traits>& ae_tm,
const Teuchos::RCP<panzer::LinearObjFactory<panzer::Traits> >& linObjFactory,
const Teuchos::RCP<const Epetra_Vector> x0 = Teuchos::null)
{
using Teuchos::RCP;
using Teuchos::rcp;
const RCP<panzer::LinearObjContainer> ghost_con = linObjFactory->buildGhostedLinearObjContainer();
const RCP<panzer::LinearObjContainer> con = linObjFactory->buildLinearObjContainer();
linObjFactory->initializeGhostedContainer(panzer::LinearObjContainer::X |
panzer::LinearObjContainer::F |
panzer::LinearObjContainer::Mat, *ghost_con);
const RCP<panzer::EpetraLinearObjContainer>
ep_con = Teuchos::rcp_dynamic_cast<panzer::EpetraLinearObjContainer>(con);
SparseTriple t;
{ // Finite-difference Jacobian. Should use graph coloring to greatly reduce
// number of evaluations. But brute force for now.
RCP<Epetra_Vector> x, f0;
for (int i = -1; ; ++i) {
linObjFactory->initializeContainer(panzer::LinearObjContainer::X | panzer::LinearObjContainer::F |
panzer::LinearObjContainer::Mat, *con);
ghost_con->initialize();
con->initialize();
const Epetra_Map& row_map = ep_con->get_A()->RowMap(), col_map = ep_con->get_A()->ColMap();
if (i == -1) {
if (Teuchos::nonnull(x0))
x = rcp(new Epetra_Vector(*x0));
else
x = rcp(new Epetra_Vector(ep_con->get_x()->Map()));
t.m = t.n = x->GlobalLength();
}
// For a linear problem, could make delta 1 to remove cancellation
// error. But I want to look at a nonlinear Robin condition, so do a true
// finite difference.
const double delta = 1e-6;
const bool i_mine = row_map.MyGID(i);
double x_prev = 0;
int i_lid = 0;
if (i_mine && i >= 0) {
i_lid = col_map.LID(i);
x_prev = (*x)[i_lid];
(*x)[i_lid] += delta;
}
ep_con->set_x(x);
panzer::AssemblyEngineInArgs input(ghost_con, ep_con);
input.alpha = 0;
input.beta = 1;
ae_tm.getAsObject<panzer::Traits::Residual>()->evaluate(input);
if (i == -1)
f0 = ep_con->get_f();
else {
const Epetra_Vector& f = *ep_con->get_f();
if (i_mine) (*x)[i_lid] = x_prev;
for (int k = 0; k < f.MyLength(); ++k) {
const double d = f[k] - (*f0)[k];
if (d == 0) continue;
t.i.push_back(row_map.GID(k));
t.j.push_back(i);
t.v.push_back(d/delta);
}
}
if (i + 1 == t.m) break;
}
}
{ // AD Jacobian.
linObjFactory->initializeContainer(panzer::LinearObjContainer::X | panzer::LinearObjContainer::F |
panzer::LinearObjContainer::Mat, *con);
ghost_con->initialize();
con->initialize();
if (Teuchos::nonnull(x0))
ep_con->set_x(rcp(new Epetra_Vector(*x0)));
else
ep_con->get_x()->PutScalar(0);
panzer::AssemblyEngineInArgs input(ghost_con, ep_con);
input.alpha = 0;
input.beta = 1;
ae_tm.getAsObject<panzer::Traits::Jacobian>()->evaluate(input);
EpetraExt::RowMatrixToMatrixMarketFile("A_ad.mm", *ep_con->get_A());
}
RCP<Epetra_CrsMatrix> A_fd;
{
const Epetra_Map& row_map = ep_con->get_A()->RowMap(), col_map = ep_con->get_A()->ColMap();
A_fd = rcp(new Epetra_CrsMatrix(Copy, row_map, 40));
for (int i = 0; i < static_cast<int>(t.v.size()); ++i)
A_fd->InsertGlobalValues(t.i[i], 1, &t.v[i], &t.j[i]);
A_fd->FillComplete();
}
EpetraExt::RowMatrixToMatrixMarketFile("A_fd.mm", *A_fd);
{ // Check error.
const double A_fd_inf_norm = A_fd->NormInf();
RCP<Epetra_CrsMatrix> D = rcp(new Epetra_CrsMatrix(Copy, ep_con->get_A()->Graph()));
D->FillComplete();
Epetra_CrsMatrix* D_ptr = D.get();
EpetraExt::MatrixMatrix::Add(*ep_con->get_A(), false, -1, *A_fd, false, 1, D_ptr);
const double D_inf_norm = D->NormInf();
return D_inf_norm / A_fd_inf_norm;
}
}
int
main (int argc, char *argv[])
{
using namespace Anasazi;
using Teuchos::RCP;
using Teuchos::rcp;
using std::endl;
#ifdef HAVE_MPI
// Initialize MPI
MPI_Init (&argc, &argv);
#endif // HAVE_MPI
// Create an Epetra communicator
#ifdef HAVE_MPI
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif // HAVE_MPI
// Create an Anasazi output manager
BasicOutputManager<double> printer;
printer.stream(Errors) << Anasazi_Version() << std::endl << std::endl;
// Get the sorting std::string from the command line
std::string which ("LM");
Teuchos::CommandLineProcessor cmdp (false, true);
cmdp.setOption("sort", &which, "Targetted eigenvalues (SM or LM).");
if (cmdp.parse (argc, argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return -1;
}
// Dimension of the matrix
//
// Discretization points in any one direction.
const int nx = 10;
// Size of matrix nx*nx
const int NumGlobalElements = nx*nx;
// Construct a Map that puts approximately the same number of
// equations on each process.
Epetra_Map Map (NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements ();
std::vector<int> MyGlobalElements (NumMyElements);
Map.MyGlobalElements (&MyGlobalElements[0]);
// Create an integer vector NumNz that is used to build the Petra
// matrix. NumNz[i] is the number of OFF-DIAGONAL terms for the
// i-th global equation on this process.
std::vector<int> NumNz (NumMyElements);
/* We are building a matrix of block structure:
| T -I |
|-I T -I |
| -I T |
| ... -I|
| -I T|
where each block is dimension nx by nx and the matrix is on the order of
nx*nx. The block T is a tridiagonal matrix.
*/
for (int i=0; i<NumMyElements; ++i) {
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1 ||
MyGlobalElements[i] == nx-1 || MyGlobalElements[i] == nx*(nx-1) ) {
NumNz[i] = 3;
}
else if (MyGlobalElements[i] < nx || MyGlobalElements[i] > nx*(nx-1) ||
MyGlobalElements[i]%nx == 0 || (MyGlobalElements[i]+1)%nx == 0) {
NumNz[i] = 4;
}
else {
NumNz[i] = 5;
}
}
// Create an Epetra_Matrix
RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, &NumNz[0]));
// Compute coefficients for discrete convection-diffution operator
const double one = 1.0;
std::vector<double> Values(4);
std::vector<int> Indices(4);
double rho = 0.0;
double h = one /(nx+1);
double h2 = h*h;
double c = 5.0e-01*rho/ h;
Values[0] = -one/h2 - c; Values[1] = -one/h2 + c; Values[2] = -one/h2; Values[3]= -one/h2;
double diag = 4.0 / h2;
int NumEntries;
for (int i=0; i<NumMyElements; ++i) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
Indices[1] = nx;
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == nx*(nx-1)) {
Indices[0] = nx*(nx-1)+1;
Indices[1] = nx*(nx-2);
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == nx-1) {
Indices[0] = nx-2;
NumEntries = 1;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = 2*nx-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == NumGlobalElements-1) {
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = nx*(nx-1)-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] < nx) {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] > nx*(nx-1)) {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i]%nx == 0) {
Indices[0] = MyGlobalElements[i]+1;
Indices[1] = MyGlobalElements[i]-nx;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if ((MyGlobalElements[i]+1)%nx == 0) {
Indices[0] = MyGlobalElements[i]-nx;
Indices[1] = MyGlobalElements[i]+nx;
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = MyGlobalElements[i]-1;
NumEntries = 1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
Indices[3] = MyGlobalElements[i]+nx;
NumEntries = 4;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
// Put in the diagonal entry
int info = A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, &MyGlobalElements[i]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
// Finish up
int info = A->FillComplete ();
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "A->FillComplete() returned info = "
<< info << " != 0." );
A->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks
// Create a identity matrix for the temporary mass matrix
RCP<Epetra_CrsMatrix> M = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, 1));
for (int i=0; i<NumMyElements; i++) {
Values[0] = one;
Indices[0] = i;
NumEntries = 1;
info = M->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "M->InsertGlobalValues() returned info = "
<< info << " != 0." );
}
// Finish up
info = M->FillComplete ();
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "M->FillComplete() returned info = "
<< info << " != 0." );
M->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks
//************************************
// Call the LOBPCG solver manager
//***********************************
//
// Variables used for the LOBPCG Method
const int nev = 10;
const int blockSize = 5;
const int maxIters = 500;
const double tol = 1.0e-8;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef MultiVecTraits<double, Epetra_MultiVector> MVT;
// Create an Epetra_MultiVector for an initial vector to start the
// solver. Note: This needs to have the same number of columns as
// the blocksize.
RCP<Epetra_MultiVector> ivec = rcp (new Epetra_MultiVector (Map, blockSize));
ivec->Random (); // fill the initial vector with random values
// Create the eigenproblem.
RCP<BasicEigenproblem<double, MV, OP> > MyProblem =
rcp (new BasicEigenproblem<double, MV, OP> (A, ivec));
// Inform the eigenproblem that the operator A is symmetric
MyProblem->setHermitian (true);
// Set the number of eigenvalues requested
MyProblem->setNEV (nev);
// Tell the eigenproblem that you are finishing passing it information.
const bool success = MyProblem->setProblem ();
if (! success) {
printer.print (Errors, "Anasazi::BasicEigenproblem::setProblem() reported an error.\n");
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return -1;
}
// Create parameter list to pass into the solver manager
Teuchos::ParameterList MyPL;
MyPL.set ("Which", which);
MyPL.set ("Block Size", blockSize);
MyPL.set ("Maximum Iterations", maxIters);
MyPL.set ("Convergence Tolerance", tol);
MyPL.set ("Full Ortho", true);
MyPL.set ("Use Locking", true);
// Create the solver manager
LOBPCGSolMgr<double, MV, OP> MySolverMan (MyProblem, MyPL);
// Solve the problem
ReturnType returnCode = MySolverMan.solve ();
// Get the eigenvalues and eigenvectors from the eigenproblem
Eigensolution<double,MV> sol = MyProblem->getSolution ();
std::vector<Value<double> > evals = sol.Evals;
RCP<MV> evecs = sol.Evecs;
// Compute residuals.
std::vector<double> normR (sol.numVecs);
if (sol.numVecs > 0) {
Teuchos::SerialDenseMatrix<int,double> T (sol.numVecs, sol.numVecs);
Epetra_MultiVector tempAevec (Map, sol.numVecs );
T.putScalar (0.0);
for (int i = 0; i < sol.numVecs; ++i) {
T(i,i) = evals[i].realpart;
}
A->Apply (*evecs, tempAevec);
MVT::MvTimesMatAddMv (-1.0, *evecs, T, 1.0, tempAevec);
MVT::MvNorm (tempAevec, normR);
}
// Print the results
std::ostringstream os;
os.setf (std::ios_base::right, std::ios_base::adjustfield);
os << "Solver manager returned "
<< (returnCode == Converged ? "converged." : "unconverged.") << endl;
os << endl;
os << "------------------------------------------------------" << endl;
os << std::setw(16) << "Eigenvalue"
<< std::setw(18) << "Direct Residual"
<< endl;
os << "------------------------------------------------------" << endl;
for (int i = 0; i < sol.numVecs; ++i) {
os << std::setw(16) << evals[i].realpart
<< std::setw(18) << normR[i] / evals[i].realpart
<< endl;
}
os << "------------------------------------------------------" << endl;
printer.print (Errors, os.str ());
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return 0;
}
void tSIMPLEPreconditionerFactory::initializeTest()
{
std::vector<int> indicies(2);
std::vector<double> row0(2),row1(2);
tolerance_ = 1.0e-13;
comm=GetComm();
const RCP<Epetra_Map> map = rcp(new Epetra_Map(2,0,*comm));
const RCP<Epetra_CrsMatrix> ptrF = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrB = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrBt = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrC = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvF = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvS = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvMass = rcp(new Epetra_CrsMatrix(Copy,*map,2));
indicies[0] = 0;
indicies[1] = 1;
// build F matrix
row0[0] = 1.0; row0[1] = 2.0;
row1[0] = 2.0; row1[1] = 1.0;
ptrF->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrF->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrF->FillComplete();
F_ = Thyra::epetraLinearOp(ptrF,"ptrF");
// build block info for block diagonal (one block)
block_starts_=arcp(new int[2],0,2);
block_gids_=arcp(new int[2],0,2);
int *block_starts=block_starts_.getRawPtr();
int *block_gids=block_gids_.getRawPtr();
block_starts[0]=0;block_starts[1]=2;
block_gids[0]=ptrF->LRID(0);
block_gids[1]=ptrF->LRID(1);
// build B matrix
row0[0] = 1.0; row0[1] = -3.0;
row1[0] = -1.0; row1[1] = 1.0;
ptrB->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrB->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrB->FillComplete();
B_ = Thyra::epetraLinearOp(ptrB,"ptrB");
// build Bt matrix
row0[0] = 1.0; row0[1] = -1.0;
row1[0] = -3.0; row1[1] = 1.0;
ptrBt->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrBt->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrBt->FillComplete();
Bt_ = Thyra::epetraLinearOp(ptrBt,"ptrBt");
// build C matrix
row0[0] = 1.0; row0[1] = 2.0;
row1[0] = 2.0; row1[1] = 1.0;
ptrC->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrC->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrC->FillComplete();
C_ = Thyra::epetraLinearOp(ptrC,"ptrC");
// build inv(F) matrix
row0[0] = -1.0/3.0; row0[1] = 2.0/3.0;
row1[0] = 2.0/3.0; row1[1] = -1.0/3.0;
ptrInvF->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvF->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvF->FillComplete();
invF_ = rcp(new StaticOpInverseFactory(Thyra::epetraLinearOp(ptrInvF,"ptrInvF")));
// build inv(Pschur) matrix
row0[0] = 0.037037037037037; row0[1] = 0.222222222222222;
row1[0] = 0.222222222222222; row1[1] = 0.333333333333333;
ptrInvS->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvS->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvS->FillComplete();
invS_ = rcp(new StaticOpInverseFactory(Thyra::epetraLinearOp(ptrInvS,"ptrInvS")));
A_ = Thyra::block2x2<double>(F_,Bt_,B_,C_,"A");
}
// build a single subblock Epetra_CrsMatrix
RCP<Epetra_CrsMatrix> buildSubBlock(int i,int j,const Epetra_CrsMatrix & A,const std::vector<std::pair<int,RCP<Epetra_Map> > > & subMaps)
{
// get the number of variables families
int numVarFamily = subMaps.size();
TEUCHOS_ASSERT(i>=0 && i<numVarFamily);
TEUCHOS_ASSERT(j>=0 && j<numVarFamily);
const Epetra_Map & gRowMap = *subMaps[i].second;
const Epetra_Map & rowMap = *Teuchos::get_extra_data<RCP<Epetra_Map> >(subMaps[i].second,"contigMap");
const Epetra_Map & colMap = *Teuchos::get_extra_data<RCP<Epetra_Map> >(subMaps[j].second,"contigMap");
int colFamilyCnt = subMaps[j].first;
// compute the number of global variables
// and the row and column block offset
int numGlobalVars = 0;
int rowBlockOffset = 0;
int colBlockOffset = 0;
for(int k=0;k<numVarFamily;k++) {
numGlobalVars += subMaps[k].first;
// compute block offsets
if(k<i) rowBlockOffset += subMaps[k].first;
if(k<j) colBlockOffset += subMaps[k].first;
}
// copy all global rows to here
Epetra_Import import(gRowMap,A.RowMap());
Epetra_CrsMatrix localA(Copy,gRowMap,0);
localA.Import(A,import,Insert);
RCP<Epetra_CrsMatrix> mat = rcp(new Epetra_CrsMatrix(Copy,rowMap,0));
// get entry information
int numMyRows = rowMap.NumMyElements();
int maxNumEntries = A.GlobalMaxNumEntries();
// for extraction
std::vector<int> indices(maxNumEntries);
std::vector<double> values(maxNumEntries);
// for insertion
std::vector<int> colIndices(maxNumEntries);
std::vector<double> colValues(maxNumEntries);
// insert each row into subblock
// let FillComplete handle column distribution
for(int localRow=0;localRow<numMyRows;localRow++) {
int numEntries = -1;
int globalRow = gRowMap.GID(localRow);
int contigRow = rowMap.GID(localRow);
TEUCHOS_ASSERT(globalRow>=0);
TEUCHOS_ASSERT(contigRow>=0);
// extract a global row copy
int err = localA.ExtractGlobalRowCopy(globalRow, maxNumEntries, numEntries, &values[0], &indices[0]);
TEUCHOS_ASSERT(err==0);
int numOwnedCols = 0;
for(int localCol=0;localCol<numEntries;localCol++) {
int globalCol = indices[localCol];
// determinate which block this column ID is in
int block = globalCol / numGlobalVars;
bool inFamily = true;
// test the beginning of the block
inFamily &= (block*numGlobalVars+colBlockOffset <= globalCol);
inFamily &= ((block*numGlobalVars+colBlockOffset+colFamilyCnt) > globalCol);
// is this column in the variable family
if(inFamily) {
int familyOffset = globalCol-(block*numGlobalVars+colBlockOffset);
colIndices[numOwnedCols] = block*colFamilyCnt + familyOffset;
colValues[numOwnedCols] = values[localCol];
numOwnedCols++;
}
}
// insert it into the new matrix
mat->InsertGlobalValues(contigRow,numOwnedCols,&colValues[0],&colIndices[0]);
}
// fill it and automagically optimize the storage
mat->FillComplete(colMap,rowMap);
return mat;
}
int main(int argc, char *argv[])
{
using std::cout;
using std::endl;
bool boolret;
int MyPID;
#ifdef HAVE_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
MyPID = Comm.MyPID();
bool testFailed = false;
bool verbose = false;
bool debug = false;
bool shortrun = false;
bool skinny = true;
bool useSA = false;
std::string which("SR");
int numElements = 100;
std::string prec("none");
int user_verbosity = 0;
int numicgs = 2;
bool success = true;
try {
CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("debug","nodebug",&debug,"Print debugging information.");
cmdp.setOption("sort",&which,"Targetted eigenvalues (SR or LR).");
cmdp.setOption("shortrun","longrun",&shortrun,"Allow only a small number of iterations.");
cmdp.setOption("skinny","hefty",&skinny,"Use a skinny (low-mem) or hefty (higher-mem) implementation of IRTR.");
cmdp.setOption("numElements",&numElements,"Number of elements in discretization.");
cmdp.setOption("prec",&prec,"Preconditioning type (""none"", ""olsen"", ""simple""");
cmdp.setOption("useSA","noSA",&useSA,"Use subspace acceleration.");
cmdp.setOption("verbosity",&user_verbosity,"Additional verbosity falgs.");
cmdp.setOption("numICGS",&numicgs,"Num ICGS iterations");
if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return -1;
}
if (debug) verbose = true;
if (prec != "olsen" && prec != "simple") {
prec = "none";
}
typedef double ScalarType;
typedef ScalarTraits<ScalarType> SCT;
typedef SCT::magnitudeType MagnitudeType;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<ScalarType,MV> MVT;
typedef Anasazi::OperatorTraits<ScalarType,MV,OP> OPT;
const ScalarType ONE = SCT::one();
if (verbose && MyPID == 0) {
cout << Anasazi::Anasazi_Version() << endl << endl;
}
// Problem information
int space_dim = 1;
std::vector<double> brick_dim( space_dim );
brick_dim[0] = 1.0;
std::vector<int> elements( space_dim );
elements[0] = numElements;
// Create problem
RCP<ModalProblem> testCase = rcp( new ModeLaplace1DQ1(Comm, brick_dim[0], elements[0]) );
//
// Get the stiffness and mass matrices
RCP<const Epetra_CrsMatrix> K = rcp( const_cast<Epetra_CrsMatrix *>(testCase->getStiffness()), false );
RCP<const Epetra_CrsMatrix> M = rcp( const_cast<Epetra_CrsMatrix *>(testCase->getMass()), false );
RCP<const Epetra_CrsMatrix> P;
if (prec != "none") {
// construct a diagonal preconditioner
Epetra_Vector D(K->RowMap(),false);
K->ExtractDiagonalCopy(D);
for (int i=0; i<D.MyLength(); ++i) {
D[i] = 1.0 / D[i];
}
RCP<Epetra_CrsMatrix> ncP = rcp( new Epetra_CrsMatrix(Epetra_DataAccess::Copy,K->RowMap(),K->RowMap(),1,true) );
for (int i=0; i<D.MyLength(); ++i) {
ncP->InsertMyValues(i,1,&D[i],&i);
}
ncP->FillComplete(true);
P = ncP;
}
//
// Create the initial vectors
int blockSize = 5;
RCP<Epetra_MultiVector> ivec = rcp( new Epetra_MultiVector(K->OperatorDomainMap(), blockSize) );
ivec->Random();
// Create eigenproblem
const int nev = 5;
RCP<Anasazi::BasicEigenproblem<ScalarType,MV,OP> > problem =
rcp( new Anasazi::BasicEigenproblem<ScalarType,MV,OP>(K,M,ivec) );
//
// Inform the eigenproblem that the operator K is symmetric
problem->setHermitian(true);
//
// Set the number of eigenvalues requested
problem->setNEV( nev );
//
// Set the preconditioner, if using one. Also, indicates the type.
if (prec != "none") {
problem->setPrec(P);
}
//
// Inform the eigenproblem that you are done passing it information
boolret = problem->setProblem();
if (boolret != true) {
if (verbose && MyPID == 0) {
cout << "Anasazi::BasicEigenproblem::SetProblem() returned with error." << endl
<< "End Result: TEST FAILED" << endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
// Set verbosity level
int verbosity = Anasazi::Errors + Anasazi::Warnings;
if (verbose) {
verbosity += Anasazi::FinalSummary + Anasazi::TimingDetails;
}
if (debug) {
verbosity += Anasazi::Debug;
}
verbosity |= user_verbosity;
// Eigensolver parameters
int maxIters;
if (shortrun) {
maxIters = 50;
}
else {
maxIters = 450;
}
MagnitudeType tol = 1.0e-6;
//
// Create parameter list to pass into the solver manager
ParameterList MyPL;
MyPL.set( "Skinny Solver", skinny);
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Which", which );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Maximum Iterations", maxIters );
MyPL.set( "Convergence Tolerance", tol );
MyPL.set( "Use SA", useSA );
MyPL.set( "Num ICGS", numicgs );
if (prec == "olsen") {
MyPL.set( "Olsen Prec", true );
}
else if (prec == "simple") {
MyPL.set( "Olsen Prec", false );
}
//
// Create the solver manager
Anasazi::RTRSolMgr<ScalarType,MV,OP> MySolverMan(problem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMan.solve();
testFailed = false;
if (returnCode != Anasazi::Converged) {
if (shortrun==false) {
testFailed = true;
}
}
/*
//
// Check that the parameters were all consumed
if (MyPL.getEntryPtr("Verbosity")->isUsed() == false ||
MyPL.getEntryPtr("Which")->isUsed() == false ||
MyPL.getEntryPtr("Skinny Solver")->isUsed() == false ||
MyPL.getEntryPtr("Block Size")->isUsed() == false ||
MyPL.getEntryPtr("Maximum Iterations")->isUsed() == false ||
MyPL.getEntryPtr("Convergence Tolerance")->isUsed() == false) {
if (verbose && MyPID==0) {
cout << "Failure! Unused parameters: " << endl;
MyPL.unused(cout);
}
}
*/
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<ScalarType,MV> sol = problem->getSolution();
std::vector<Anasazi::Value<ScalarType> > evals = sol.Evals;
RCP<MV> evecs = sol.Evecs;
int numev = sol.numVecs;
if (numev > 0) {
std::ostringstream os;
os.setf(std::ios::scientific, std::ios::floatfield);
os.precision(6);
// Check the problem against the analytical solutions
if (verbose) {
double *revals = new double[numev];
for (int i=0; i<numev; i++) {
revals[i] = evals[i].realpart;
}
bool smallest = false;
if (which == "SR") {
smallest = true;
}
testCase->eigenCheck( *evecs, revals, 0, smallest );
delete [] revals;
}
// Compute the direct residual
std::vector<ScalarType> normV( numev );
SerialDenseMatrix<int,ScalarType> T(numev,numev);
for (int i=0; i<numev; i++) {
T(i,i) = evals[i].realpart;
}
RCP<MV> Mvecs = MVT::Clone( *evecs, numev ),
Kvecs = MVT::Clone( *evecs, numev );
OPT::Apply( *K, *evecs, *Kvecs );
OPT::Apply( *M, *evecs, *Mvecs );
MVT::MvTimesMatAddMv( -ONE, *Mvecs, T, ONE, *Kvecs );
// compute M-norm of residuals
OPT::Apply( *M, *Kvecs, *Mvecs );
MVT::MvDot( *Mvecs, *Kvecs, normV );
os << "Number of iterations performed in RTR_test.exe: " << MySolverMan.getNumIters() << endl
<< "Direct residual norms computed in RTR_test.exe" << endl
<< std::setw(20) << "Eigenvalue" << std::setw(20) << "Residual(M)" << endl
<< "----------------------------------------" << endl;
for (int i=0; i<numev; i++) {
if ( SCT::magnitude(evals[i].realpart) != SCT::zero() ) {
normV[i] = SCT::magnitude( SCT::squareroot( normV[i] ) / evals[i].realpart );
}
else {
normV[i] = SCT::magnitude( SCT::squareroot( normV[i] ) );
}
os << std::setw(20) << evals[i].realpart << std::setw(20) << normV[i] << endl;
if ( normV[i] > tol ) {
testFailed = true;
}
}
if (verbose && MyPID==0) {
cout << endl << os.str() << endl;
}
}
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true,cout,success);
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
if (testFailed || success==false) {
if (verbose && MyPID==0) {
cout << "End Result: TEST FAILED" << endl;
}
return -1;
}
//
// Default return value
//
if (verbose && MyPID==0) {
cout << "End Result: TEST PASSED" << endl;
}
return 0;
}
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp; //
//
// MPI initialization using Teuchos
//
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
//
// Parameters
//
GlobalOrdinal numGlobalElements = 256; // problem size
//
// Construct the problem
//
// Construct a Map that puts approximately the same number of equations on each processor
const Epetra_Map map(numGlobalElements, 0, comm);
// Get update list and number of local equations from newly created map.
const size_t numMyElements = map.NumMyElements();
const GlobalOrdinal* myGlobalElements = map.MyGlobalElements();
// Create a CrsMatrix using the map, with a dynamic allocation of 3 entries per row
RCP<Epetra_CrsMatrix> A = rcp(new Epetra_CrsMatrix(Copy, map, 3));
// Add rows one-at-a-time
for (size_t i = 0; i < numMyElements; i++) {
if (myGlobalElements[i] == 0) {
//TODO: should be rewritten in an Epetra style
A->InsertGlobalValues(myGlobalElements[i], 2,
Teuchos::tuple<Scalar> (2.0, -1.0).getRawPtr(),
Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i], myGlobalElements[i] +1).getRawPtr());
}
else if (myGlobalElements[i] == numGlobalElements - 1) {
A->InsertGlobalValues(myGlobalElements[i], 2,
Teuchos::tuple<Scalar> (-1.0, 2.0).getRawPtr(),
Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i] -1, myGlobalElements[i]).getRawPtr());
}
else {
A->InsertGlobalValues(myGlobalElements[i], 3,
Teuchos::tuple<Scalar> (-1.0, 2.0, -1.0).getRawPtr(),
Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i] -1, myGlobalElements[i], myGlobalElements[i] +1).getRawPtr());
}
}
// Complete the fill, ask that storage be reallocated and optimized
A->FillComplete();
//
// Construct a multigrid preconditioner
//
// Turns a Epetra_CrsMatrix into a MueLu::Matrix
RCP<Xpetra::CrsMatrix<SC, LO, GO, NO, LMO> > mueluA_ = rcp(new Xpetra::EpetraCrsMatrix(A)); //TODO: should not be needed
RCP<Xpetra::Matrix <SC, LO, GO, NO, LMO> > mueluA = rcp(new Xpetra::CrsMatrixWrap<SC, LO, GO, NO, LMO>(mueluA_));
// Multigrid Hierarchy
RCP<Hierarchy> H = rcp(new Hierarchy(mueluA));
H->setVerbLevel(Teuchos::VERB_HIGH);
// Multigrid setup phase (using default parameters)
H->Setup();
//
// Define RHS / LHS
//
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(map));
RCP<Epetra_Vector> B = rcp(new Epetra_Vector(map));
X->PutScalar((Scalar) 0.0);
B->SetSeed(846930886); B->Random();
#ifndef HAVE_MUELU_BELOS
//
// Use AMG directly as an iterative solver (not as a preconditionner)
//
int nIts = 9;
// Wrap Epetra Vectors into Xpetra Vectors
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(X));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(B));
H->Iterate(*mueluB, nIts, *mueluX);
// Print relative residual norm
ST::magnitudeType residualNorms = Utils::ResidualNorm(*mueluA, *mueluX, *mueluB)[0];
if (comm.MyPID() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
#else // HAVE_MUELU_BELOS
//
// Solve Ax = b using AMG as a preconditioner in Belos
//
// Matrix and Multivector type that will be used with Belos
typedef Epetra_MultiVector MV;
typedef Belos::OperatorT<MV> OP;
// Define Operator and Preconditioner
RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(mueluA)); // Turns a Xpetra::Matrix object into a Belos operator
RCP<OP> belosPrec = rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 20;
double tol = 1e-4;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = solver->solve();
// Get the number of iterations for this solve.
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
// Compute actual residuals.
int numrhs=1;
bool badRes = false;
std::vector<SC> actual_resids(numrhs);
std::vector<SC> rhs_norm(numrhs);
RCP<Epetra_MultiVector > resid = rcp(new Epetra_MultiVector(map, numrhs));
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply(*belosOp, *X, *resid);
MVT::MvAddMv(-1.0, *resid, 1.0, *B, *resid);
MVT::MvNorm(*resid, actual_resids);
MVT::MvNorm(*B, rhs_norm);
std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for (int i = 0; i < numrhs; i++) {
SC actRes = actual_resids[i]/rhs_norm[i];
std::cout <<"Problem " << i << " : \t" << actRes <<std::endl;
if (actRes > tol) { badRes = true; }
}
// Check convergence
if (ret != Belos::Converged || badRes) {
std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
return EXIT_FAILURE;
}
std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif // HAVE_MUELU_BELOS
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm Comm;
#endif
// My MPI process rank.
const int MyPID = Comm.MyPID();
// "/Users/sakashitatatsuya/Downloads/barista_trunk_slepc/sample/hamiltonian_matrix.ip"
std::ifstream ifs(argv[1]);
alps::Parameters params(ifs);
Teuchos::oblackholestream blackHole;
std::ostream& out = (MyPID == 0) ? std::cout : blackHole;
barista::Hamiltonian<> hamiltonian(params);
matrix_type matrix(hamiltonian.dimension(), hamiltonian.dimension());
hamiltonian.fill<double>(matrix);
int m,n;
int N;
m = n = N = hamiltonian.dimension();
//std::cout << matrix << std::endl;
std::ofstream ofs;
if (MyPID==0) {
ofs.open("anasazi_time.txt");
if (!ofs) {
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
}
//Teuchos::ParameterList GaleriList;
using Teuchos::RCP;
using Teuchos::rcp;
typedef Teuchos::ScalarTraits<double> STS;
const double one = STS::one();
const double zero = STS::zero();
// The problem is defined on a 2D grid, global size is nx * nx.
//int nx = N;
//GaleriList.set("n", nx * nx);
//GaleriList.set("nx", nx);
//GaleriList.set("ny", nx);
//Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( Galeri::CreateMap("Linear", Comm, GaleriList) );
//Teuchos::RCP<Epetra_RowMatrix> A = Teuchos::rcp( Galeri::CreateCrsMatrix("Laplace2D", &*Map, GaleriList) );
// Construct a Map that puts approximately the same number of rows
// of the matrix A on each processor.
Epetra_Map RowMap (N, 0, Comm);
Epetra_Map ColMap (N, 0, Comm);
// Get update list and number of local equations from newly created Map.
const int NumMyRowElements = RowMap.NumMyElements ();
std::vector<int> MyGlobalRowElements (NumMyRowElements);
RowMap.MyGlobalElements (&MyGlobalRowElements[0]);
// Create an Epetra_CrsMatrix using the given row map.
RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Copy, RowMap, n));
// We use info to catch any errors that may have happened during // matrix assembly, and report them globally. We do this so that // the MPI processes won't call FillComplete() unless they all // successfully filled their parts of the matrix.
int info = 0;
try {
//
// Compute coefficients for the discrete integral operator.
//
std::vector<double> Values (n);
std::vector<int> Indices (n);
//const double inv_mp1 = one / (m+1);
//const double inv_np1 = one / (n+1);
int count;
//for (int i = 0; i < n; ++i) {
// Indices[i] = i;
//}
for (int i = 0; i < NumMyRowElements; ++i) {
count =0;
for (int j = 0; j < n; ++j) {
if (matrix(MyGlobalRowElements[i],j)!=0) {
Values[count] = matrix(MyGlobalRowElements[i],j);
Indices[count] = j;
count++;
}
}
info = A->InsertGlobalValues (MyGlobalRowElements[i], count,
&Values[0], &Indices[0]);
// Make sure that the insertion succeeded. Teuchos'
// TEST_FOR_EXCEPTION macro gives a nice error message if the
// thrown exception isn't caught. We'll report this on the
// offending MPI process.
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error, "Failed to insert n="
<< n << " global value" << (n != 1 ? "s" : "")
<< " in row " << MyGlobalRowElements[i]
<< " of the matrix." );
*/
} // for i = 0...
// Call FillComplete on the matrix. Since the matrix isn't square,
// we have to give FillComplete the domain and range maps, which in
// this case are the column resp. row maps.
info = A->FillComplete (ColMap, RowMap);
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error,
"FillComplete failed with INFO = " << info << ".");
*/
info = A->OptimizeStorage();
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error,
"OptimizeStorage failed with INFO = " << info << ".");
*/
} catch (std::runtime_error& e) {
// If multiple MPI processes are reporting errors, sometimes
// forming the error message as a string and then writing it to
// the output stream prevents messages from different processes
// from being interleaved.
std::ostringstream os;
os << "*** Error on MPI process " << MyPID << ": " << e.what();
cerr << os.str() << endl;
if (info == 0)
info = -1; // All procs will share info later on.
}
// Variables used for the Block Davidson Method
const int nev = 5;
const int blockSize = 5;
const int numBlocks = 8;
const int maxRestarts = 500;
const double tol = 1.0e-8;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<double, Epetra_MultiVector> MVT;
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
//
//Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(*Map, blockSize) );
Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(ColMap, blockSize) );
ivec->Random();
// Create the eigenproblem.
Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > problem =
Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(A, ivec) );
// Inform the eigenproblem that the operator A is symmetric
problem->setHermitian(true);
// Set the number of eigenvalues requested
problem->setNEV( nev );
// Inform the eigenproblem that you are finishing passing it information
bool boolret = problem->setProblem();
if (boolret != true) {
std::cout<<"Anasazi::BasicEigenproblem::setProblem() returned an error." << std::endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return -1;
}
// Create parameter list to pass into the solver manager
Teuchos::ParameterList anasaziPL;
anasaziPL.set( "Which", "LM" );
anasaziPL.set( "Block Size", blockSize );
anasaziPL.set( "Maximum Iterations", 500 );
anasaziPL.set( "Convergence Tolerance", tol );
anasaziPL.set( "Verbosity", Anasazi::Errors+Anasazi::Warnings+Anasazi::TimingDetails+Anasazi::FinalSummary );
// Create the solver manager
Anasazi::LOBPCGSolMgr<double, MV, OP> anasaziSolver(problem, anasaziPL);
// Solve the problem
double start, end;
MPI_Barrier(MPI_COMM_WORLD);
start = MPI_Wtime();
Anasazi::ReturnType returnCode = anasaziSolver.solve();
MPI_Barrier(MPI_COMM_WORLD);
end = MPI_Wtime();
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<double,MV> sol = problem->getSolution();
std::vector<Anasazi::Value<double> > evals = sol.Evals;
Teuchos::RCP<MV> evecs = sol.Evecs;
// Compute residuals.
std::vector<double> normR(sol.numVecs);
Teuchos::SerialDenseMatrix<int,double> T(sol.numVecs, sol.numVecs);
Epetra_MultiVector tempAevec( ColMap, sol.numVecs );
T.putScalar(0.0);
for (int i=0; i<sol.numVecs; i++) {
T(i,i) = evals[i].realpart;
}
A->Apply( *evecs, tempAevec );
MVT::MvTimesMatAddMv( -1.0, *evecs, T, 1.0, tempAevec );
MVT::MvNorm( tempAevec, normR );
if (MyPID == 0) {
// Print the results
std::cout<<"Solver manager returned " << (returnCode == Anasazi::Converged ? "converged." : "unconverged.") << std::endl;
std::cout<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
std::cout<<std::setw(16)<<"Eigenvalue"
<<std::setw(18)<<"Direct Residual"
<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
for (int i=0; i<sol.numVecs; i++) {
std::cout<<std::setw(16)<<evals[i].realpart
<<std::setw(18)<<normR[i]/evals[i].realpart
<<std::endl;
}
std::cout<<"------------------------------------------------------"<<std::endl;
}
// Print out the map and matrices
//ColMap.Print (out);
//A->Print (cout);
//RowMap.Print (cout);
double time;
int iter;
if (MyPID==0) {
iter = anasaziSolver.getNumIters();
Teuchos::Array<Teuchos::RCP<Teuchos::Time> > timer = anasaziSolver.getTimers();
Teuchos::RCP<Teuchos::Time> _timerSolve = timer[0];
cout << "timerSolve=" << _timerSolve << endl;
time = end - start;
cout << "time=" << time << endl;
ofs << "time=" << time << endl;
cout << "iter=" << iter << endl;
ofs << "iter=" << iter << endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return 0;
}
void tJacobi2x2PreconditionerFactory::initializeTest()
{
std::vector<int> indicies(2);
std::vector<double> row0(2),row1(2);
tolerance_ = 1.0e-14;
comm = rcp(new Epetra_SerialComm());
const RCP<Epetra_Map> map = rcp(new Epetra_Map(2,0,*comm));
const RCP<Epetra_CrsMatrix> ptrF = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrG = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrD = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrC = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvF = rcp(new Epetra_CrsMatrix(Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvC = rcp(new Epetra_CrsMatrix(Copy,*map,2));
indicies[0] = 0;
indicies[1] = 1;
// build F matrix
row0[0] = 1.0; row0[1] = 2.0;
row1[0] = 2.0; row1[1] = 1.0;
ptrF->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrF->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrF->FillComplete();
F_ = Thyra::epetraLinearOp(ptrF,"ptrF");
// build D matrix
row0[0] = 1.0; row0[1] = -3.0;
row1[0] = -1.0; row1[1] = 1.0;
ptrD->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrD->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrD->FillComplete();
D_ = Thyra::epetraLinearOp(ptrD,"ptrD");
// build G matrix
row0[0] = 1.0; row0[1] = -1.0;
row1[0] = -3.0; row1[1] = 1.0;
ptrG->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrG->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrG->FillComplete();
G_ = Thyra::epetraLinearOp(ptrG,"ptrG");
// build C matrix
row0[0] = 9.0; row0[1] = 2.0;
row1[0] = 6.0; row1[1] = 5.0;
ptrC->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrC->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrC->FillComplete();
C_ = Thyra::epetraLinearOp(ptrC,"ptrC");
// build inv(F) matrix
row0[0] = -1.0/3.0; row0[1] = 2.0/3.0;
row1[0] = 2.0/3.0; row1[1] = -1.0/3.0;
ptrInvF->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvF->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvF->FillComplete();
invF_ = Thyra::epetraLinearOp(ptrInvF,"ptrInvF");
// build inv(C) matrix
row0[0] = 0.151515151515151; row0[1] = -0.060606060606061;
row1[0] = -0.181818181818182; row1[1] = 0.272727272727273;
ptrInvC->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvC->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvC->FillComplete();
invC_ = Thyra::epetraLinearOp(ptrInvC,"ptrInvC");
A_ = Thyra::block2x2<double>(F_,G_,D_,C_,"A");
}
Teuchos::Array<double>
test_original_matrix_free_epetra(
const Teuchos::Array<int> & var_degree ,
const int nGrid ,
const int iterCount ,
const bool print_flag ,
const bool test_block ,
const bool check )
{
typedef double value_type ;
typedef Stokhos::OneDOrthogPolyBasis<int,value_type> abstract_basis_type;
typedef Stokhos::LegendreBasis<int,value_type> basis_type;
typedef Stokhos::CompletePolynomialBasis<int,value_type> product_basis_type;
typedef Stokhos::Sparse3Tensor<int,value_type> Cijk_type;
using Teuchos::rcp;
using Teuchos::RCP;
using Teuchos::Array;
using Teuchos::ParameterList;
// Create a communicator for Epetra objects
RCP<const Epetra_Comm> globalComm;
#ifdef HAVE_MPI
RCP<const Epetra_MpiComm> globalMpiComm =
rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
//globalMpiComm->Print(std::cout);
globalComm = globalMpiComm;
#else
RCP<const Epetra_SerialComm> globalSerialComm =
rcp(new Epetra_SerialComm);
//globalSerialComm->Print(std::cout);
globalComm = globalSerialComm;
#endif
//int MyPID = globalComm->MyPID();
// Create Stochastic Galerkin basis and expansion
const size_t num_KL = var_degree.size();
Array< RCP<const abstract_basis_type> > bases(num_KL);
for (size_t i=0; i<num_KL; i++)
bases[i] = rcp(new basis_type(var_degree[i],true));
RCP<const product_basis_type> basis =
rcp(new product_basis_type(bases, 1e-12));
const size_t stoch_length = basis->size();
RCP<Cijk_type> Cijk = basis->computeTripleProductTensor();
// Create stochastic parallel distribution
ParameterList parallelParams;
RCP<Stokhos::ParallelData> sg_parallel_data =
rcp(new Stokhos::ParallelData(basis, Cijk, globalComm, parallelParams));
RCP<const EpetraExt::MultiComm> sg_comm =
sg_parallel_data->getMultiComm();
RCP<const Epetra_Comm> app_comm =
sg_parallel_data->getSpatialComm();
RCP<const Stokhos::EpetraSparse3Tensor> epetraCijk =
sg_parallel_data->getEpetraCijk();
RCP<const Epetra_BlockMap> stoch_row_map =
epetraCijk->getStochasticRowMap();
//------------------------------
// Generate FEM graph:
Teuchos::Array< Teuchos::Array<int> > fem_graph ;
const size_t fem_length = nGrid * nGrid * nGrid ;
generate_fem_graph( nGrid , fem_graph );
//------------------------------
// Generate Epetra objects
RCP<const Epetra_Map> x_map = rcp(new Epetra_Map(static_cast<int>(fem_length), 0, *app_comm));
RCP<const Epetra_Map> sg_x_map =
rcp(EpetraExt::BlockUtility::GenerateBlockMap(
*x_map, *stoch_row_map, *sg_comm));
RCP<Epetra_CrsGraph> graph = rcp(new Epetra_CrsGraph(Copy, *x_map, 27));
int *my_GIDs = x_map->MyGlobalElements();
int num_my_GIDs = x_map->NumMyElements();
for (int i=0; i<num_my_GIDs; ++i) {
int row = my_GIDs[i];
int num_indices = fem_graph[row].size();
int *indices = &(fem_graph[row][0]);
graph->InsertGlobalIndices(row, num_indices, indices);
}
graph->FillComplete();
int nnz = graph->NumGlobalNonzeros();
RCP<ParameterList> sg_op_params = rcp(new ParameterList);
RCP<Stokhos::MatrixFreeOperator> sg_A =
rcp(new Stokhos::MatrixFreeOperator(sg_comm, basis, epetraCijk,
x_map, x_map, sg_x_map, sg_x_map,
sg_op_params));
RCP<Epetra_BlockMap> sg_A_overlap_map =
rcp(new Epetra_LocalMap(
static_cast<int>(stoch_length), 0, *(sg_parallel_data->getStochasticComm())));
RCP< Stokhos::EpetraOperatorOrthogPoly > A_sg_blocks =
rcp(new Stokhos::EpetraOperatorOrthogPoly(
basis, sg_A_overlap_map, x_map, x_map, sg_x_map, sg_comm));
for (unsigned int i=0; i<stoch_length; i++) {
RCP<Epetra_CrsMatrix> A = rcp(new Epetra_CrsMatrix(Copy, *graph));
A->FillComplete();
A->PutScalar(1.0);
A_sg_blocks->setCoeffPtr(i, A);
}
sg_A->setupOperator(A_sg_blocks);
RCP<Stokhos::EpetraVectorOrthogPoly> sg_x =
rcp(new Stokhos::EpetraVectorOrthogPoly(
basis, stoch_row_map, x_map, sg_x_map, sg_comm));
RCP<Stokhos::EpetraVectorOrthogPoly> sg_y =
rcp(new Stokhos::EpetraVectorOrthogPoly(
basis, stoch_row_map, x_map, sg_x_map, sg_comm));
sg_x->init(1.0);
sg_y->init(0.0);
// Time apply
Teuchos::Time clock("apply") ;
clock.start();
for ( int iter = 0 ; iter < iterCount ; ++iter ) {
sg_A->Apply( *(sg_x->getBlockVector()), *(sg_y->getBlockVector()) );
}
clock.stop();
double seconds_per_iter =
clock.totalElapsedTime() / ((double) iterCount );
// Averge time across proc's
double average_seconds_per_iter;
globalComm->SumAll(&seconds_per_iter, &average_seconds_per_iter, 1);
average_seconds_per_iter /= globalComm->NumProc();
// Compute number of fem mat-vec's
int n_apply = 0;
int n_add = 0;
for (Cijk_type::k_iterator k_it=Cijk->k_begin();
k_it!=Cijk->k_end(); ++k_it) {
for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
j_it != Cijk->j_end(k_it); ++j_it) {
++n_apply;
for (Cijk_type::kji_iterator i_it = Cijk->i_begin(j_it);
i_it != Cijk->i_end(j_it); ++i_it) {
++n_add;
}
}
}
const double flops = 1.0e-9*(2.0*static_cast<double>(n_apply)*nnz +
static_cast<double>(n_add)*fem_length);
//------------------------------
Teuchos::Array<double> perf(8);
perf[0] = stoch_length;
perf[1] = fem_length;
perf[2] = stoch_length * fem_length;
perf[3] = Cijk->num_entries();
perf[4] = nnz;
perf[5] = average_seconds_per_iter ;
perf[6] = flops/average_seconds_per_iter;
perf[7] = flops;
return perf;
}
void tLSCStablePreconditionerFactory::initializeTest()
{
std::vector<int> indicies(2);
std::vector<double> row0(2),row1(2);
tolerance_ = 1.0e-13;
comm = rcp(new Epetra_SerialComm());
const RCP<Epetra_Map> map = rcp(new Epetra_Map(2,0,*comm));
const RCP<Epetra_CrsMatrix> ptrF = rcp(new Epetra_CrsMatrix(Epetra_DataAccess::Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrB = rcp(new Epetra_CrsMatrix(Epetra_DataAccess::Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrBt = rcp(new Epetra_CrsMatrix(Epetra_DataAccess::Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvF = rcp(new Epetra_CrsMatrix(Epetra_DataAccess::Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvS = rcp(new Epetra_CrsMatrix(Epetra_DataAccess::Copy,*map,2));
const RCP<Epetra_CrsMatrix> ptrInvMass = rcp(new Epetra_CrsMatrix(Epetra_DataAccess::Copy,*map,2));
indicies[0] = 0;
indicies[1] = 1;
// build F matrix
row0[0] = 1.0; row0[1] = 2.0;
row1[0] = 2.0; row1[1] = 1.0;
ptrF->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrF->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrF->FillComplete();
F_ = Thyra::epetraLinearOp(ptrF,"ptrF");
// build B matrix
row0[0] = 1.0; row0[1] = -3.0;
row1[0] = -1.0; row1[1] = 1.0;
ptrB->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrB->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrB->FillComplete();
B_ = Thyra::epetraLinearOp(ptrB,"ptrB");
// build Bt matrix
row0[0] = 1.0; row0[1] = -1.0;
row1[0] = -3.0; row1[1] = 1.0;
ptrBt->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrBt->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrBt->FillComplete();
Bt_ = Thyra::epetraLinearOp(ptrBt,"ptrBt");
// build inv(F) matrix
row0[0] = -1.0/3.0; row0[1] = 2.0/3.0;
row1[0] = 2.0/3.0; row1[1] = -1.0/3.0;
ptrInvF->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvF->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvF->FillComplete();
invF_ = Thyra::epetraLinearOp(ptrInvF,"ptrInvF");
// build inv(Mass) matrix
row0[0] = 3.0; row0[1] = 0.0;
row1[0] = 0.0; row1[1] = 2.0;
ptrInvMass->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvMass->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvMass->FillComplete();
invMass_ = Thyra::epetraLinearOp(ptrInvMass,"ptrInvMass");
// build inv(Pschur) matrix
row0[0] = 0.208333333333333; row0[1] = 0.375000000000000;
row1[0] = 0.375000000000000; row1[1] = 0.875000000000000;
ptrInvS->InsertGlobalValues(0,2,&row0[0],&indicies[0]);
ptrInvS->InsertGlobalValues(1,2,&row1[0],&indicies[0]);
ptrInvS->FillComplete();
invBQBt_ = Thyra::scale<double>(-1.0,Thyra::epetraLinearOp(ptrInvS,"ptrInvS"));
A_ = Thyra::block2x2<double>(F_,Bt_,B_,Thyra::zero(Bt_->range(),B_->domain()),"A");
}
// int
// main(int argc, char * argv[])
TEUCHOS_UNIT_TEST(tALOperator, test)
{
// Build communicator
#ifdef HAVE_MPI
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
// Get process information
int myPID = Comm.MyPID();
out << "MPI_PID = " << myPID << ", UNIX_PID = " << getpid() << std::endl;
// Maps.
int dim = 2, numVel = 3, numPre = 2, errCode;
Epetra_Map mapVel(numVel, 0, Comm), mapPre(numPre, 0, Comm);
Epetra_Map mapAll(numVel * dim + numPre, 0, Comm);
// Reorder.
std::vector<int> reorderedVec;
int numMyLen = mapVel.NumMyElements();
int * myGlb;
myGlb = mapVel.MyGlobalElements();
for(int i = 0; i < dim; i++)
for(int j = 0; j < numMyLen; j++)
reorderedVec.push_back(myGlb[j] + numVel * i);
numMyLen = mapPre.NumMyElements();
myGlb = mapPre.MyGlobalElements();
for(int j = 0; j < numMyLen; j++)
reorderedVec.push_back(myGlb[j] + numVel * dim);
Teuchos::RCP < Epetra_Map > mapReorder = Teuchos::rcp(
new Epetra_Map(-1, reorderedVec.size(), &reorderedVec[0], 0, Comm));
Teuchos::RCP < Epetra_Import > importReorder = Teuchos::rcp(new Epetra_Import(*mapReorder, mapAll));
std::vector<std::vector<int> > blockedVec;
numMyLen = mapVel.NumMyElements();
myGlb = mapVel.MyGlobalElements();
for(int i = 0; i < dim; i++)
{
reorderedVec.clear();
for(int j = 0; j < numMyLen; j++)
reorderedVec.push_back(myGlb[j] + numVel * i);
blockedVec.push_back(reorderedVec);
}
numMyLen = mapPre.NumMyElements();
myGlb = mapPre.MyGlobalElements();
reorderedVec.clear();
for(int j = 0; j < numMyLen; j++)
reorderedVec.push_back(myGlb[j] + numVel * dim);
blockedVec.push_back(reorderedVec);
// Read matrices and vector.
Epetra_CrsMatrix *ptrMat = 0, *ptrMp = 0;
TEUCHOS_ASSERT(EpetraExt::MatrixMarketFileToCrsMatrix("data/tOpMat.mm", mapAll, ptrMat)==0);
TEUCHOS_ASSERT(EpetraExt::MatrixMarketFileToCrsMatrix("data/tOpMp.mm", mapPre, ptrMp)==0);
LinearOp lpMp = Thyra::epetraLinearOp(Teuchos::rcpFromRef(*ptrMp));
// This vector is computed by Matlab for comparison.
Epetra_Vector *ptrExact = 0;
TEUCHOS_ASSERT(EpetraExt::MatrixMarketFileToVector("data/tOpRhs.mm", mapAll, ptrExact)==0);
// Reorder matrix.
RCP < Epetra_CrsMatrix > mat = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *mapReorder, ptrMat->GlobalMaxNumEntries()));
errCode = mat->Import(*ptrMat, *importReorder, Insert);
errCode = mat->FillComplete();
// Build augmented Lagrangian-based operator.
Teko::NS::ALOperator al(blockedVec, mat, lpMp);
// Initialize vectors.
Epetra_Vector x(*mapReorder, false), b(*mapReorder, false);
x.PutScalar(1.0);
b.PutScalar(0.0);
// Apply operator.
al.Apply(x, b);
// Compare computed vector and exact vector.
b.Update(-1.0, *ptrExact, 1.0);
double norm2;
b.Norm2(&norm2);
if(norm2 < 1.0e-15)
{
out << "Test:ALOperator: Passed." << std::endl;
errCode = 0;
}
else
{
out << "Test:ALOperator: Failed." << std::endl;
errCode = -1;
}
delete ptrMat;
delete ptrMp;
delete ptrExact;
TEST_ASSERT(errCode==0);
}
int main (int argc, char *argv[])
{
// Initialize MPI
Teuchos::GlobalMPISession (&argc, &argv, NULL);
// Create output stream. (Handy for multicore output.)
const RCP<Teuchos::FancyOStream> out =
Teuchos::VerboseObjectBase::getDefaultOStream();
// Create a communicator for Epetra objects
#ifdef HAVE_MPI
RCP<Epetra_MpiComm> eComm =
rcp<Epetra_MpiComm> (new Epetra_MpiComm (MPI_COMM_WORLD));
#else
RCP<Epetra_SerialComm> eComm =
rcp<Epetra_SerialComm> (new Epetra_SerialComm());
#endif
bool success = true;
try {
// Create map.
// Do strong scaling tests, so keep numGlobalElements independent of
// the number of processes.
int numGlobalElements = 5e7;
int indexBase = 0;
RCP<Epetra_Map> map =
rcp(new Epetra_Map (numGlobalElements, indexBase, *eComm));
//// Create map with overlay.
//int numMyOverlapNodes = 3;
//// Get an approximation of my nodes.
//int numMyElements = numGlobalElements / eComm->NumProc();
//int startIndex = eComm->MyPID() * numMyElements;
//// Calculate the resulting number of total nodes.
//int numTotalNodes = numMyElements * eComm->NumProc();
//// Add one node to the first numGlobalElements-numTotalNodes processes.
//if (eComm->MyPID() < numGlobalElements - numTotalNodes)
//{
// numMyElements++;
// startIndex += eComm->MyPID();
//}
//else
//{
// startIndex += numGlobalElements - numTotalNodes;
//}
//Teuchos::Array<int> indices(numMyElements);
//for (int k = 0; k<numMyElements; k++)
// indices[k] = startIndex + k;
//std::cout << numGlobalElements << std::endl;
//std::cout << numMyElements << std::endl;
//RCP<Epetra_Map> overlapMap =
// rcp(new Epetra_Map (numGlobalElements, numMyElements, indices.getRawPtr(), indexBase, *eComm));
//overlapMap->Print(std::cout);
//throw 1;
// tests on one vector
RCP<Epetra_Vector> u = rcp(new Epetra_Vector(*map));
u->Random();
RCP<Teuchos::Time> meanValueTime =
Teuchos::TimeMonitor::getNewTimer("Vector::MeanValue");
{
Teuchos::TimeMonitor tm(*meanValueTime);
double meanVal;
TEUCHOS_ASSERT_EQUALITY(0, u->MeanValue(&meanVal));
}
RCP<Teuchos::Time> maxValueTime =
Teuchos::TimeMonitor::getNewTimer("Vector::MaxValue");
{
Teuchos::TimeMonitor tm(*maxValueTime);
double maxValue;
TEUCHOS_ASSERT_EQUALITY(0, u->MaxValue(&maxValue));
}
RCP<Teuchos::Time> minValueTime =
Teuchos::TimeMonitor::getNewTimer("Vector::MinValue");
{
Teuchos::TimeMonitor tm(*minValueTime);
double minValue;
TEUCHOS_ASSERT_EQUALITY(0, u->MinValue(&minValue));
}
RCP<Teuchos::Time> norm1Time =
Teuchos::TimeMonitor::getNewTimer("Vector::Norm1");
{
Teuchos::TimeMonitor tm(*norm1Time);
double norm1;
TEUCHOS_ASSERT_EQUALITY(0, u->Norm1(&norm1));
}
RCP<Teuchos::Time> norm2Time =
Teuchos::TimeMonitor::getNewTimer("Vector::Norm2");
{
Teuchos::TimeMonitor tm(*norm2Time);
double norm2;
TEUCHOS_ASSERT_EQUALITY(0, u->Norm2(&norm2));
}
RCP<Teuchos::Time> normInfTime =
Teuchos::TimeMonitor::getNewTimer("Vector::NormInf");
{
Teuchos::TimeMonitor tm(*normInfTime);
double normInf;
TEUCHOS_ASSERT_EQUALITY(0, u->NormInf(&normInf));
}
RCP<Teuchos::Time> scaleTime =
Teuchos::TimeMonitor::getNewTimer("Vector::Scale");
{
Teuchos::TimeMonitor tm(*scaleTime);
double alpha = 0.5;
TEUCHOS_ASSERT_EQUALITY(0, u->Scale(0.5));
}
// tests involving two vectors
RCP<Epetra_Vector> v = rcp(new Epetra_Vector(*map));
v->Random();
RCP<Teuchos::Time> dotTime =
Teuchos::TimeMonitor::getNewTimer("Vector::Dot");
{
Teuchos::TimeMonitor tm(*dotTime);
double dot;
TEUCHOS_ASSERT_EQUALITY(0, u->Dot(*v, &dot));
}
RCP<Teuchos::Time> multiplyTime =
Teuchos::TimeMonitor::getNewTimer("Vector::Multiply");
{
Teuchos::TimeMonitor tm(*multiplyTime);
TEUCHOS_ASSERT_EQUALITY(0, u->Multiply(1.0, *u, *v, 1.0));
}
RCP<Teuchos::Time> updateTime =
Teuchos::TimeMonitor::getNewTimer("Vector::Update");
{
Teuchos::TimeMonitor tm(*updateTime);
TEUCHOS_ASSERT_EQUALITY(0, u->Update(1.0, *v, 1.0));
}
// matrix-vector tests
// diagonal test matrix
RCP<Epetra_CrsMatrix> D =
rcp(new Epetra_CrsMatrix(Copy, *map, 1));
for (int k = 0; k < map->NumMyElements(); k++) {
int col = map->GID(k);
double val = 1.0 / (col+1);
//TEUCHOS_ASSERT_EQUALITY(0, D->InsertMyValues(k, 1, &val, &col));
TEUCHOS_ASSERT_EQUALITY(0, D->InsertGlobalValues(col, 1, &val, &col));
}
TEUCHOS_ASSERT_EQUALITY(0, D->FillComplete());
// tridiagonal test matrix
RCP<Epetra_CrsMatrix> T =
rcp(new Epetra_CrsMatrix(Copy, *map, 3));
for (int k = 0; k < map->NumMyElements(); k++) {
int row = map->GID(k);
if (row > 0) {
int col = row-1;
double val = -1.0;
//TEUCHOS_ASSERT_EQUALITY(0, T->InsertMyValues(k, 1, &val, &col));
TEUCHOS_ASSERT_EQUALITY(0, T->InsertGlobalValues(row, 1, &val, &col));
}
{
int col = row;
double val = 2.0;
//TEUCHOS_ASSERT_EQUALITY(0, T->InsertMyValues(k, 1, &val, &col));
TEUCHOS_ASSERT_EQUALITY(0, T->InsertGlobalValues(row, 1, &val, &col));
}
if (row < numGlobalElements-1) {
int col = row+1;
double val = -1.0;
//TEUCHOS_ASSERT_EQUALITY(0, T->InsertMyValues(k, 1, &val, &col));
TEUCHOS_ASSERT_EQUALITY(0, T->InsertGlobalValues(row, 1, &val, &col));
}
}
TEUCHOS_ASSERT_EQUALITY(0, T->FillComplete());
// start timings
RCP<Teuchos::Time> mNorm1Time =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::Norm1");
{
Teuchos::TimeMonitor tm(*mNorm1Time);
double dNorm1 = D->NormOne();
double tNorm1 = T->NormOne();
}
RCP<Teuchos::Time> mNormInfTime =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::NormInf");
{
Teuchos::TimeMonitor tm(*mNormInfTime);
double dNormInf = D->NormInf();
double tNormInf = T->NormInf();
}
RCP<Teuchos::Time> mNormFrobTime =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::NormFrobenius");
{
Teuchos::TimeMonitor tm(*mNormFrobTime);
double dNormFrob = D->NormFrobenius();
double tNormFrob = T->NormFrobenius();
}
RCP<Teuchos::Time> mScaleTime =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::Scale");
{
Teuchos::TimeMonitor tm(*mScaleTime);
TEUCHOS_ASSERT_EQUALITY(0, D->Scale(2.0));
TEUCHOS_ASSERT_EQUALITY(0, T->Scale(2.0));
}
RCP<Teuchos::Time> leftScaleTime =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::LeftScale");
{
Teuchos::TimeMonitor tm(*leftScaleTime);
TEUCHOS_ASSERT_EQUALITY(0, D->LeftScale(*v));
TEUCHOS_ASSERT_EQUALITY(0, T->LeftScale(*v));
}
RCP<Teuchos::Time> rightScaleTime =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::RightScale");
{
Teuchos::TimeMonitor tm(*rightScaleTime);
TEUCHOS_ASSERT_EQUALITY(0, D->RightScale(*v));
TEUCHOS_ASSERT_EQUALITY(0, T->RightScale(*v));
}
RCP<Teuchos::Time> applyTime =
Teuchos::TimeMonitor::getNewTimer("CrsMatrix::Apply");
{
Teuchos::TimeMonitor tm(*applyTime);
TEUCHOS_ASSERT_EQUALITY(0, D->Apply(*u, *v));
TEUCHOS_ASSERT_EQUALITY(0, T->Apply(*u, *v));
}
// print timing data
Teuchos::TimeMonitor::summarize();
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, *out, success);
return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
//
// Diagonal: 0=no change, 1=eliminate entry
// from the map for the largest row element in process 0
// 2=add diagonal entries to the matrix, with a zero value
// (assume row map contains all diagonal entries).
//
// ReindexRowMap:
// 0=no change, 1= add 2 (still contiguous), 2=non-contiguous
//
// ReindexColMap
// 0=same as RowMap, 1=add 4 - Different From RowMap, but contiguous)
//
// RangeMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
// DomainMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
RCP<Epetra_CrsMatrix> NewMatNewMap(Epetra_CrsMatrix& In,
int Diagonal,
int ReindexRowMap,
int ReindexColMap,
int RangeMapType,
int DomainMapType
)
{
//
// If we are making no change, return the original matrix (which has a linear map)
//
#if 0
std::cout << __FILE__ << "::" << __LINE__ << " "
<< Diagonal << " "
<< ReindexRowMap << " "
<< ReindexColMap << " "
<< RangeMapType << " "
<< DomainMapType << " " << std::endl ;
#endif
if ( Diagonal + ReindexRowMap + ReindexColMap + RangeMapType + DomainMapType == 0 ) {
RCP<Epetra_CrsMatrix> ReturnOrig = rcp( &In, false );
return ReturnOrig ;
}
//
// Diagonal==2 is used for a different purpose -
// Making sure that the diagonal of the matrix is non-empty.
// Note: The diagonal must exist in In.RowMap().
//
if ( Diagonal == 2 ) {
assert( ReindexRowMap==0 && ReindexColMap == 0 ) ;
}
int (*RowPermute)(int in) = 0;
int (*ColPermute)(int in) = 0;
assert( Diagonal >= 0 && Diagonal <= 2 );
assert( ReindexRowMap>=0 && ReindexRowMap<=2 );
assert( ReindexColMap>=0 && ReindexColMap<=1 );
assert( RangeMapType>=0 && RangeMapType<=3 );
assert( DomainMapType>=0 && DomainMapType<=3 );
Epetra_Map DomainMap = In.DomainMap();
Epetra_Map RangeMap = In.RangeMap();
Epetra_Map ColMap = In.ColMap();
Epetra_Map RowMap = In.RowMap();
int NumMyRowElements = RowMap.NumMyElements();
int NumMyColElements = ColMap.NumMyElements();
int NumMyRangeElements = RangeMap.NumMyElements();
int NumMyDomainElements = DomainMap.NumMyElements();
int NumGlobalRowElements = RowMap.NumGlobalElements();
int NumGlobalColElements = ColMap.NumGlobalElements();
int NumGlobalRangeElements = RangeMap.NumGlobalElements();
int NumGlobalDomainElements = DomainMap.NumGlobalElements();
assert( NumGlobalRangeElements == NumGlobalDomainElements ) ;
std::vector<int> MyGlobalRowElements( NumMyRowElements ) ;
std::vector<int> NumEntriesPerRow( NumMyRowElements ) ;
std::vector<int> MyPermutedGlobalRowElements( NumMyRowElements ) ;
std::vector<int> MyGlobalColElements( NumMyColElements ) ;
std::vector<int> MyPermutedGlobalColElements( NumMyColElements ) ; // Used to create the column map
std::vector<int> MyPermutedGlobalColElementTable( NumMyColElements ) ; // To convert local indices to global
std::vector<int> MyGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyPermutedGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyGlobalDomainElements( NumMyDomainElements ) ;
std::vector<int> MyPermutedGlobalDomainElements( NumMyDomainElements ) ;
RowMap.MyGlobalElements(&MyGlobalRowElements[0]);
ColMap.MyGlobalElements(&MyGlobalColElements[0]);
RangeMap.MyGlobalElements(&MyGlobalRangeElements[0]);
DomainMap.MyGlobalElements(&MyGlobalDomainElements[0]);
switch( ReindexRowMap ) {
case 0:
RowPermute = &NoPermute ;
break;
case 1:
RowPermute = &SmallRowPermute ;
break;
case 2:
RowPermute = BigRowPermute ;
break;
}
switch( ReindexColMap ) {
case 0:
ColPermute = RowPermute ;
break;
case 1:
ColPermute = &SmallColPermute ;
break;
}
//
// Create Serial Range and Domain Maps based on the permuted indexing
//
int nlocal = 0;
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements;
std::vector<int> AllIDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDs[i] = (*RowPermute)( i ) ;
Epetra_Map SerialRangeMap( -1, nlocal, &AllIDs[0], 0, In.Comm());
std::vector<int> AllIDBs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDBs[i] = (*ColPermute)( i ) ;
Epetra_Map SerialDomainMap( -1, nlocal, &AllIDBs[0], 0, In.Comm());
//
// Create Bizarre Range and Domain Maps based on the permuted indexing
// These are nearly serial, having all but one element on process 0
// The goal here is to make sure that we can use Domain and Range maps
// that are neither serial, nor distributed in the normal manner.
//
std::vector<int> AllIDCs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDCs[i] = (*ColPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDCs[0] = (*ColPermute)( NumGlobalRangeElements - 1 );
}
}
int iam = In.Comm().MyPID();
Epetra_Map BizarreDomainMap( -1, nlocal, &AllIDCs[0], 0, In.Comm());
std::vector<int> AllIDDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDDs[i] = (*RowPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDDs[0] = (*RowPermute)( NumGlobalRangeElements -1 ) ;
}
}
Epetra_Map BizarreRangeMap( -1, nlocal, &AllIDDs[0], 0, In.Comm());
//
// Compute the column map
//
// If Diagonal==1, remove the column corresponding to the last row owned
// by process 0. Removing this column from a tridiagonal matrix, leaves
// a disconnected, but non-singular matrix.
//
int NumMyColElementsOut = 0 ;
int NumGlobalColElementsOut ;
if ( Diagonal == 1 )
NumGlobalColElementsOut = NumGlobalColElements-1;
else
NumGlobalColElementsOut = NumGlobalColElements;
if ( Diagonal == 1 && iam==0 ) {
for ( int i=0; i < NumMyColElements ; i++ ) {
if ( MyGlobalColElements[i] != MyGlobalRowElements[NumMyRowElements-1] ) {
MyPermutedGlobalColElements[NumMyColElementsOut++] =
(*ColPermute)( MyGlobalColElements[i] ) ;
}
}
assert( NumMyColElementsOut == NumMyColElements-1 );
} else {
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElements[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
NumMyColElementsOut = NumMyColElements ;
if ( Diagonal == 2 ) {
// For each row, make sure that the column map has this row in it,
// if it doesn't, add it to the column map.
// Note: MyPermutedGlobalColElements == MyGlobalColElements when
// Diagonal==2 because ( Diagonal == 2 ) implies:
// ReindexRowMap==0 && ReindexColMap == 0 - see assert above
for ( int i=0; i < NumMyRowElements ; i++ ) {
bool MissingDiagonal = true;
for ( int j=0; j < NumMyColElements; j++ ) {
if ( MyGlobalRowElements[i] == MyGlobalColElements[j] ) {
MissingDiagonal = false;
}
}
if ( MissingDiagonal ) {
MyPermutedGlobalColElements.resize(NumMyColElements+1);
MyPermutedGlobalColElements[NumMyColElementsOut] = MyGlobalRowElements[i];
NumMyColElementsOut++;
}
}
In.Comm().SumAll(&NumMyColElementsOut,&NumGlobalColElementsOut,1);
}
}
//
// These tables are used both as the permutation tables and to create the maps.
//
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElementTable[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
for ( int i=0; i < NumMyRowElements ; i++ )
MyPermutedGlobalRowElements[i] =
(*RowPermute)( MyGlobalRowElements[i] ) ;
for ( int i=0; i < NumMyRangeElements ; i++ )
MyPermutedGlobalRangeElements[i] =
(*RowPermute)( MyGlobalRangeElements[i] ) ;
for ( int i=0; i < NumMyDomainElements ; i++ )
MyPermutedGlobalDomainElements[i] =
(*ColPermute)( MyGlobalDomainElements[i] ) ;
RCP<Epetra_Map> PermutedRowMap =
rcp( new Epetra_Map( NumGlobalRowElements, NumMyRowElements,
&MyPermutedGlobalRowElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedColMap =
rcp( new Epetra_Map( NumGlobalColElementsOut, NumMyColElementsOut,
&MyPermutedGlobalColElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedRangeMap =
rcp( new Epetra_Map( NumGlobalRangeElements, NumMyRangeElements,
&MyPermutedGlobalRangeElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedDomainMap =
rcp( new Epetra_Map( NumGlobalDomainElements, NumMyDomainElements,
&MyPermutedGlobalDomainElements[0], 0, In.Comm() ) );
//
// These vectors are filled and then passed to InsertGlobalValues
//
std::vector<int> ThisRowIndices( In.MaxNumEntries() );
std::vector<double> ThisRowValues( In.MaxNumEntries() );
std::vector<int> PermutedGlobalColIndices( In.MaxNumEntries() );
//std::cout << __FILE__ << "::" <<__LINE__ << std::endl ;
RCP<Epetra_CrsMatrix> Out =
rcp( new Epetra_CrsMatrix( Copy, *PermutedRowMap, *PermutedColMap, 0 ) );
for (int i=0; i<NumMyRowElements; i++)
{
int NumIndicesThisRow = 0;
assert( In.ExtractMyRowCopy( i,
In.MaxNumEntries(),
NumIndicesThisRow,
&ThisRowValues[0],
&ThisRowIndices[0] ) == 0 ) ;
for (int j = 0 ; j < NumIndicesThisRow ; j++ )
{
PermutedGlobalColIndices[j] = MyPermutedGlobalColElementTable[ ThisRowIndices[j] ] ;
}
bool MissingDiagonal = false;
if ( Diagonal==2 ) {
//
assert( MyGlobalRowElements[i] == MyPermutedGlobalRowElements[i] );
MissingDiagonal = true;
for( int j =0 ; j < NumIndicesThisRow ; j++ ) {
if ( PermutedGlobalColIndices[j] == MyPermutedGlobalRowElements[i] ) {
MissingDiagonal = false ;
}
}
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< " MyPermutedGlobalRowElements[i] = " << MyPermutedGlobalRowElements[i]
<< " MissingDiagonal = " << MissingDiagonal << std::endl ;
#endif
}
if ( MissingDiagonal ) {
ThisRowValues.resize(NumIndicesThisRow+1) ;
ThisRowValues[NumIndicesThisRow] = 0.0;
PermutedGlobalColIndices.resize(NumIndicesThisRow+1);
PermutedGlobalColIndices[NumIndicesThisRow] = MyPermutedGlobalRowElements[i] ;
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< "NumIndicesThisRow = " << NumIndicesThisRow
<< "ThisRowValues[NumIndicesThisRow = " << ThisRowValues[NumIndicesThisRow]
<< " PermutedGlobalColIndices[NumIndcesThisRow] = " << PermutedGlobalColIndices[NumIndicesThisRow]
<< std::endl ;
#endif
NumIndicesThisRow++ ;
}
assert( Out->InsertGlobalValues( MyPermutedGlobalRowElements[i],
NumIndicesThisRow,
&ThisRowValues[0],
&PermutedGlobalColIndices[0] ) >= 0 );
}
//
Epetra_LocalMap ReplicatedMap( NumGlobalRangeElements, 0, In.Comm() );
RCP<Epetra_Map> OutRangeMap ;
RCP<Epetra_Map> OutDomainMap ;
switch( RangeMapType ) {
case 0:
OutRangeMap = PermutedRangeMap ;
break;
case 1:
OutRangeMap = rcp(&SerialRangeMap, false);
break;
case 2:
OutRangeMap = rcp(&BizarreRangeMap, false);
break;
case 3:
OutRangeMap = rcp(&ReplicatedMap, false);
break;
}
// switch( DomainMapType ) {
switch( DomainMapType ) {
case 0:
OutDomainMap = PermutedDomainMap ;
break;
case 1:
OutDomainMap = rcp(&SerialDomainMap, false);
break;
case 2:
OutDomainMap = rcp(&BizarreDomainMap, false);
break;
case 3:
OutDomainMap = rcp(&ReplicatedMap, false);
break;
}
#if 0
assert(Out->FillComplete( *PermutedDomainMap, *PermutedRangeMap )==0);
#else
assert(Out->FillComplete( *OutDomainMap, *OutRangeMap )==0);
#endif
#if 0
std::cout << __FILE__ << "::" << __LINE__ << std::endl ;
Out->Print( std::cout ) ;
#endif
return Out;
}
