Teuchos::RCP<Epetra_Map>
Albany::SolutionResponseFunction::
buildCulledMap(const Epetra_Map& x_map,
const Teuchos::Array<int>& keepDOF) const
{
int numKeepDOF = std::accumulate(keepDOF.begin(), keepDOF.end(), 0);
int Neqns = keepDOF.size();
int N = x_map.NumMyElements(); // x_map is map for solution vector
TEUCHOS_ASSERT( !(N % Neqns) ); // Assume that all the equations for
// a given node are on the assigned
// processor. I.e. need to ensure
// that N is exactly Neqns-divisible
int nnodes = N / Neqns; // number of fem nodes
int N_new = nnodes * numKeepDOF; // length of local x_new
int *gids = x_map.MyGlobalElements(); // Fill local x_map into gids array
Teuchos::Array<int> gids_new(N_new);
int idx = 0;
for ( int inode = 0; inode < N/Neqns ; ++inode) // For every node
for ( int ieqn = 0; ieqn < Neqns; ++ieqn ) // Check every dof on the node
if ( keepDOF[ieqn] == 1 ) // then want to keep this dof
gids_new[idx++] = gids[(inode*Neqns)+ieqn];
// end cull
Teuchos::RCP<Epetra_Map> x_new_map =
Teuchos::rcp( new Epetra_Map( -1, N_new, &gids_new[0], 0, x_map.Comm() ) );
return x_new_map;
}
// =============================================================================
Teuchos::RCP<Epetra_Map>
VIO::EpetraMesh::Reader::
createComplexValuesMap_ ( const Epetra_Map & nodesMap
) const
{
// get view for the global indices of the global elements
int numMyElements = nodesMap.NumMyElements();
Teuchos::ArrayRCP<int> myGlobalElements( numMyElements );
nodesMap.MyGlobalElements( myGlobalElements.getRawPtr() );
// Construct the map in such a way that all complex entries on processor K
// are split up into real and imaginary part, which will both reside on
// processor K again.
int numMyComplexElements = 2*numMyElements;
Teuchos::ArrayRCP<int> myComplexGlobalElements ( numMyComplexElements );
for ( int k = 0; k < numMyElements; k++ )
{
myComplexGlobalElements[2*k ] = 2 * myGlobalElements[k];
myComplexGlobalElements[2*k+1] = 2 * myGlobalElements[k] + 1;
}
return Teuchos::rcp ( new Epetra_Map ( -1,
myComplexGlobalElements.size(),
myComplexGlobalElements.getRawPtr(),
nodesMap.IndexBase(),
nodesMap.Comm()
)
);
}
TridiagonalCrsMatrix(
const Epetra_Map & Map,
double a,
double diag,
double c
) :
Epetra_CrsMatrix(Copy,Map,3)
{
// global number of rows
int NumGlobalElements = Map.NumGlobalElements();
// local number of rows
int NumMyElements = Map.NumMyElements();
// get update list
int * MyGlobalElements = new int [NumMyElements];
Map.MyGlobalElements( MyGlobalElements );
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values = new double[2];
Values[0] = a;
Values[1] = c;
int *Indices = new int[2];
int NumEntries;
for( int i=0 ; i<NumMyElements; ++i ) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
NumEntries = 1;
} else if (MyGlobalElements[i] == NumGlobalElements-1) {
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
} else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices);
// Put in the diagonal entry
InsertGlobalValues(MyGlobalElements[i], 1, &diag, MyGlobalElements+i);
}
// Finish up
FillComplete();
delete [] MyGlobalElements;
delete [] Values;
delete [] Indices;
}
//==============================================================================
int Komplex_LinearProblem::MakeKomplexMap(const Epetra_Map & Map, Teuchos::RCP<Epetra_Map> & KMap) {
if (Map.LinearMap()) {
KMap = Teuchos::rcp(new Epetra_Map(-1, Map.NumMyElements()*2, Map.IndexBase(), Map.Comm()));
}
else {
Epetra_IntSerialDenseVector KMapGIDs(Map.NumMyElements()*2);
int * myGlobalElements = Map.MyGlobalElements();
for (int i=0; i<Map.NumMyElements(); i++) {
KMapGIDs[2*i] = 2*myGlobalElements[i];
KMapGIDs[2*i+1] = 2*myGlobalElements[i]+1;
}
KMap = Teuchos::rcp(new Epetra_Map(-1, KMapGIDs.Length(), KMapGIDs.Values(), Map.IndexBase(), Map.Comm()));
}
return(0);
}
//EpetraMap_To_TpetraMap: takes in Epetra_Map object, converts it to its equivalent Tpetra::Map object,
//and returns an RCP pointer to this Tpetra::Map
Teuchos::RCP<const Tpetra_Map> Petra::EpetraMap_To_TpetraMap(const Epetra_Map& epetraMap_,
const Teuchos::RCP<const Teuchos::Comm<int> >& commT_)
{
const std::size_t numElements = Teuchos::as<std::size_t>(epetraMap_.NumMyElements());
const auto indexBase = Teuchos::as<GO>(epetraMap_.IndexBase());
if (epetraMap_.DistributedGlobal() || epetraMap_.Comm().NumProc() == Teuchos::OrdinalTraits<int>::one()) {
Teuchos::Array<Tpetra_GO> indices(numElements);
int *epetra_indices = epetraMap_.MyGlobalElements();
for(LO i=0; i < numElements; i++)
indices[i] = epetra_indices[i];
const Tpetra::global_size_t computeGlobalElements = Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid();
return Teuchos::rcp(new Tpetra_Map(computeGlobalElements, indices, indexBase, commT_));
} else {
return Teuchos::rcp(new Tpetra_Map(numElements, indexBase, commT_, Tpetra::LocallyReplicated));
}
}
bool Vector2MATLAB( const Epetra_Vector & v,
const Epetra_Map & Map)
{
int MyPID = Map.Comm().MyPID();
int NumProc = Map.Comm().NumProc();
int MyLength = v.MyLength();
int GlobalLength = v.GlobalLength();
// print out on cout if no filename is provided
// write on file the dimension of the matrix
if( MyPID == 0 ) cout << "v = zeros(" << GlobalLength << ")\n";
// get update list
int * MyGlobalElements = Map.MyGlobalElements( );
int Row;
for( int Proc=0 ; Proc<NumProc ; ++Proc ) {
if( MyPID == Proc ) {
cout << "% On proc " << Proc << ": ";
cout << MyLength << " rows of ";
cout << GlobalLength << " elements\n";
for( Row=0 ; Row<MyLength ; ++Row ) {
cout << "b(" << MyGlobalElements[Row]
<< ") = " << v[Row] << ";\n";
}
if( MyPID == NumProc-1 ) {
cout << "% End of vector\n";
}
}
Map.Comm().Barrier();
}
return true;
} /* Vector2MATLAB */
// FIXME long long
Epetra_Map
Epetra_Util::Create_OneToOne_Map(const Epetra_Map& usermap,
bool high_rank_proc_owns_shared)
{
//if usermap is already 1-to-1 then we'll just return a copy of it.
if (usermap.IsOneToOne()) {
Epetra_Map newmap(usermap);
return(newmap);
}
int myPID = usermap.Comm().MyPID();
Epetra_Directory* directory = usermap.Comm().CreateDirectory(usermap);
int numMyElems = usermap.NumMyElements();
const int* myElems = usermap.MyGlobalElements();
int* owner_procs = new int[numMyElems];
directory->GetDirectoryEntries(usermap, numMyElems, myElems, owner_procs,
0, 0, high_rank_proc_owns_shared);
//we'll fill a list of map-elements which belong on this processor
int* myOwnedElems = new int[numMyElems];
int numMyOwnedElems = 0;
for(int i=0; i<numMyElems; ++i) {
int GID = myElems[i];
int owner = owner_procs[i];
if (myPID == owner) {
myOwnedElems[numMyOwnedElems++] = GID;
}
}
Epetra_Map one_to_one_map(-1, numMyOwnedElems, myOwnedElems,
usermap.IndexBase(), usermap.Comm());
delete [] myOwnedElems;
delete [] owner_procs;
delete directory;
return(one_to_one_map);
}
/* Computes the approximate Schur complement for the wide separator
using guided probing*/
Teuchos::RCP<Epetra_CrsMatrix> computeSchur_GuidedProbing
(
shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure
Epetra_Map *localDRowMap
)
{
int i;
double relative_thres = config->relative_threshold;
Epetra_CrsMatrix *G = ssym->G.getRawPtr();
Epetra_CrsMatrix *R = ssym->R.getRawPtr();
Epetra_LinearProblem *LP = ssym->LP.getRawPtr();
Amesos_BaseSolver *solver = ssym->Solver.getRawPtr();
Ifpack_Preconditioner *ifSolver = ssym->ifSolver.getRawPtr();
Epetra_CrsMatrix *C = ssym->C.getRawPtr();
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
Epetra_Map CrMap = C->RowMap();
int *c_rows = CrMap.MyGlobalElements();
int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
int c_localElems = CrMap.NumMyElements();
int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
Epetra_Map RrMap = R->RowMap();
int *r_rows = RrMap.MyGlobalElements();
int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
int r_localElems = RrMap.NumMyElements();
int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map C_localRMap (-1, c_localElems, c_rows, 0, LComm);
Epetra_Map C_localCMap (-1, c_localcolElems, c_cols, 0, LComm);
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
Epetra_Map R_localRMap (-1, r_localElems, r_rows, 0, LComm);
Epetra_Map R_localCMap (-1, r_localcolElems, r_cols, 0, LComm);
//cout << "#local rows" << g_localElems << "#non zero local cols" << c_localcolElems << endl;
#ifdef DEBUG
cout << "DEBUG MODE" << endl;
int nrows = C->RowMap().NumMyElements();
assert(nrows == localDRowMap->NumGlobalElements());
int gids[nrows], gids1[nrows];
C_localRMap.MyGlobalElements(gids);
localDRowMap->MyGlobalElements(gids1);
cout << "Comparing R's domain map with D's row map" << endl;
for (int i = 0; i < nrows; i++)
{
assert(gids[i] == gids1[i]);
}
#endif
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
//cout << "Creating local matrices" << endl;
int err;
Epetra_CrsMatrix localC(Copy, C_localRMap, C->MaxNumEntries(), false);
for (i = 0; i < c_localElems ; i++)
{
gid = c_rows[i];
err = C->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1,
indices1);
assert (err == 0);
//if (nentries1 > 0) // TODO: Later
//{
err = localC.InsertGlobalValues(gid, nentries1, values1, indices1);
assert (err == 0);
//}
}
localC.FillComplete(G_localRMap, C_localRMap);
//cout << "Created local C matrix" << endl;
Epetra_CrsMatrix localR(Copy, R_localRMap, R->MaxNumEntries(), false);
for (i = 0; i < r_localElems ; i++)
{
gid = r_rows[i];
R->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
localR.InsertGlobalValues(gid, nentries1, values1, indices1);
}
localR.FillComplete(*localDRowMap, R_localRMap);
//cout << "Created local R matrix" << endl;
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
cout << "Created local G matrix" << endl;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, ssym, &localG, &localR, LP, solver,
ifSolver, &localC, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
Teuchos::RCP<Epetra_CrsGraph> lSGraph = Teuchos::RCP<Epetra_CrsGraph> (
new Epetra_CrsGraph(Copy, G_localRMap, maxentries));
if (data->num_compute % config->reset_iter == 0)
{
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[g_localElems];
int *indices = new int[g_localElems];
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec(G_localRMap, nvectors);
Epetra_MultiVector Scol(G_localRMap, nvectors);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
probevec.PutScalar(0.0); // TODO: Move it out
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
// Not much of use for Shasta 2x2 .. Later.
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[cindex] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0)
{
dropped++;
//cout << "vecvalues[j]" << vecvalues[j] <<
// " max" << maxvalue[k] << endl;
}
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < g_localElems ; i++)
{
// TODO: Can move the next two decalarations outside the loop
Epetra_MultiVector probevec(G_localRMap, 1);
Epetra_MultiVector Scol(G_localRMap, 1);
probevec.PutScalar(0.0);
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[i], 0, 1.0);
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[i] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
probeop.PrintTimingInfo();
Sbar->FillComplete();
lSGraph->FillComplete();
data->localSbargraph = lSGraph;
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
cout << "#dropped entries" << dropped << endl;
delete[] values;
delete[] indices;
delete[] maxvalue;
}
else
{
if (((data->num_compute-1) % config->reset_iter) == 0)
{
// We recomputed the Schur complement with dropping for the last
// compute. Reset the prober with the new orthogonal vectors for
// the Sbar from the previous iteration.
Teuchos::ParameterList pList;
Teuchos::RCP<Isorropia::Epetra::Prober> gprober =
Teuchos::RCP<Isorropia::Epetra::Prober> (new
Isorropia::Epetra::Prober(
data->localSbargraph.getRawPtr(), pList, false));
gprober->color();
data->guided_prober = gprober;
}
// Use the prober to probe the probeop for the sparsity pattern
// add that to Sbar and call Fill complete
int nvectors = data->guided_prober->getNumOrthogonalVectors();
cout << "Number of Orthogonal Vectors for guided probing" << nvectors
<< endl;
probeop.ResetTempVectors(nvectors);
Teuchos::RCP<Epetra_CrsMatrix> blockdiag_Sbar =
data->guided_prober->probe(probeop);
int maxentries = blockdiag_Sbar->GlobalMaxNumEntries();
int *indices = new int[maxentries];
double *values = new double[maxentries];
int numentries;
for (int i = 0; i < blockdiag_Sbar->NumGlobalRows() ; i++)
{
int gid = blockdiag_Sbar->GRID(i);
blockdiag_Sbar->ExtractGlobalRowCopy(gid, maxentries, numentries,
values, indices);
Sbar->InsertGlobalValues(gid, numentries, values, indices);
}
Sbar->FillComplete();
delete[] indices;
delete[] values;
}
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
return Sbar;
}
/* Apply an identity matrix to the Schur complement operator. Drop the entries
entries using a relative threshold. Assemble the result in a Crs Matrix
which will be our approximate Schur complement.
*/
Teuchos::RCP<Epetra_CrsMatrix> computeApproxSchur(shylu_config *config,
shylu_symbolic *sym,
Epetra_CrsMatrix *G, Epetra_CrsMatrix *R,
Epetra_LinearProblem *LP, Amesos_BaseSolver *solver,
Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C,
Epetra_Map *localDRowMap)
{
double relative_thres = config->relative_threshold;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, sym, G, R, LP, solver, ifSolver, C,
localDRowMap, nvectors);
// Get row map
Epetra_Map rMap = G->RowMap();
int *rows = rMap.MyGlobalElements();
int totalElems = rMap.NumGlobalElements();
int localElems = rMap.NumMyElements();
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
int prefixSum;
G->Comm().ScanSum(&localElems, &prefixSum, 1);
//cout << " prefixSum" << prefixSum << endl;
// Start the index in prefixSum-localElems
int *mySGID = new int[totalElems]; // vector of size Schur complement !
int *allSGID = new int[totalElems]; // vector of size Schur complement !
int i, j;
for (i = 0, j = 0; i < totalElems ; i++)
{
if (i >= prefixSum - localElems && i < prefixSum)
{
mySGID[i] = rows[j];
j++;
}
else
{
mySGID[i] = 0;
}
allSGID[i] = 0;
}
C->Comm().SumAll(mySGID, allSGID, totalElems);
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time to Compute RowIDS" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
// Now everyone knows the GIDs in the Schur complement
//cout << rMap << endl;
j = 0;
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, rMap, localElems));
int nentries;
double *values = new double[localElems]; // Need to adjust this for more
int *indices = new int[localElems]; // than one vector
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("Apply time");
#endif
int findex = totalElems / nvectors ;
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
Epetra_MultiVector probevec(rMap, nvectors);
Epetra_MultiVector Scol(rMap, nvectors);
probevec.PutScalar(0.0);
int cindex;
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
if (cindex >= prefixSum - localElems && cindex < prefixSum)
{
probevec.ReplaceGlobalValue(allSGID[cindex], k, 1.0);
}
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (j = 0 ; j < localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (allSGID[cindex] == rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[cindex];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[cindex];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < totalElems ; i++)
{
Epetra_MultiVector probevec(rMap, 1); // TODO: Try doing more than one
Epetra_MultiVector Scol(rMap, 1); // vector at a time
probevec.PutScalar(0.0);
if (i >= prefixSum - localElems && i < prefixSum)
{
probevec.ReplaceGlobalValue(allSGID[i], 0, 1.0);
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (j = 0 ; j < localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (allSGID[i] == rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[i];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[i];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
Sbar->FillComplete();
cout << "#dropped entries" << dropped << endl;
delete[] allSGID;
delete[] mySGID;
delete[] values;
delete[] indices;
delete[] maxvalue;
return Sbar;
}
int CreateTridi(Epetra_CrsMatrix& A)
{
Epetra_Map Map = A.RowMap();
int NumMyElements = Map.NumMyElements();
int NumGlobalElements = Map.NumGlobalElements();
int * MyGlobalElements = new int[NumMyElements];
Map.MyGlobalElements(MyGlobalElements);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values = new double[3];
int *Indices = new int[3];
int NumEntries;
for (int i=0; i<NumMyElements; i++)
{
if (MyGlobalElements[i]==0)
{
Indices[0] = 0;
Indices[1] = 1;
Values[0] = 2.0;
Values[1] = -1.0;
NumEntries = 2;
}
else if (MyGlobalElements[i] == NumGlobalElements-1)
{
Indices[0] = NumGlobalElements-1;
Indices[1] = NumGlobalElements-2;
Values[0] = 2.0;
Values[1] = -1.0;
NumEntries = 2;
}
else
{
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i];
Indices[2] = MyGlobalElements[i]+1;
Values[0] = -1.0;
Values[1] = 2.0;
Values[2] = -1.0;
NumEntries = 3;
}
assert(A.InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices)==0);
// Put in the diagonal entry
// assert(A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i])==0);
}
// Finish up
assert(A.FillComplete()==0);
delete[] MyGlobalElements;
delete[] Values;
delete[] Indices;
return 0;
}
/* Computes the approximate Schur complement for the wide separator */
Teuchos::RCP<Epetra_CrsMatrix> computeApproxWideSchur(shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
Epetra_CrsMatrix *G, Epetra_CrsMatrix *R,
Epetra_LinearProblem *LP, Amesos_BaseSolver *solver,
Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C,
Epetra_Map *localDRowMap)
{
int i;
double relative_thres = config->relative_threshold;
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
//Epetra_Map CrMap = C->RowMap();
//int *c_rows = CrMap.MyGlobalElements();
//int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
//int c_localElems = CrMap.NumMyElements();
//int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
//Epetra_Map RrMap = R->RowMap();
//int *r_rows = RrMap.MyGlobalElements();
//int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
//int r_localElems = RrMap.NumMyElements();
//int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
//cout << "Created local G matrix" << endl;
int nvectors = 16;
/*ShyLU_Probing_Operator probeop(&localG, &localR, LP, solver, &localC,
localDRowMap, nvectors);*/
ShyLU_Local_Schur_Operator probeop(config, ssym, &localG, R, LP, solver,
ifSolver, C, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[nvectors];
int *indices = new int[nvectors];
double *vecvalues;
#ifdef SHYLU_DEBUG
// mfh 25 May 2015: Don't declare this variable if it's not used.
// It's only used if SHYLU_DEBUG is defined.
int dropped = 0;
#endif // SHYLU_DEBUG
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec (G_localRMap, nvectors);
Epetra_MultiVector Scol (G_localRMap, nvectors);
probevec.PutScalar(0.0);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
// Set the probevec to find block columns of S.
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
// Reset the probevec to all zeros.
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
probevec.ReplaceGlobalValue(g_rows[cindex], k, 0.0);
}
Scol.MaxValue(maxvalue);
nentries = 0;
for (int j = 0 ; j < g_localElems ; j++)
{
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
vecvalues = Scol[k];
if ((g_rows[cindex] == g_rows[j]) ||
(abs(vecvalues[j]/maxvalue[k]) > relative_thres))
// diagonal entry or large entry.
{
values[nentries] = vecvalues[j];
indices[nentries++] = g_rows[cindex];
}
#ifdef SHYLU_DEBUG
else if (vecvalues[j] != 0.0)
{
dropped++;
}
#endif // SHYLU_DEBUG
}
Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
nentries = 0;
}
}
if (i < g_localElems)
{
nvectors = g_localElems - i;
probeop.ResetTempVectors(nvectors);
Epetra_MultiVector probevec1 (G_localRMap, nvectors);
Epetra_MultiVector Scol1 (G_localRMap, nvectors);
probevec1.PutScalar(0.0);
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec1.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec1, Scol1);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol1.MaxValue(maxvalue);
nentries = 0;
for (int j = 0 ; j < g_localElems ; j++)
{
//cout << "MAX" << maxvalue << endl;
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
vecvalues = Scol1[k];
//nentries = 0; // inserting one entry in each row for now
if ((g_rows[cindex] == g_rows[j]) ||
(abs(vecvalues[j]/maxvalue[k]) > relative_thres))
// diagonal entry or large entry.
{
values[nentries] = vecvalues[j];
indices[nentries++] = g_rows[cindex];
}
#ifdef SHYLU_DEBUG
else if (vecvalues[j] != 0.0)
{
dropped++;
}
#endif // SHYLU_DEBUG
}
Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
nentries = 0;
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime()
<< endl;
ftime.reset();
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
probeop.PrintTimingInfo();
#endif
Sbar->FillComplete();
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
#ifdef SHYLU_DEBUG
cout << "#dropped entries" << dropped << endl;
#endif
delete[] values;
delete[] indices;
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
delete[] maxvalue;
return Sbar;
}
void
LOCA::Epetra::AugmentedOp::buildExtendedMap(const Epetra_BlockMap& uMap,
Epetra_Map*& eMapPtr,
bool buildImporter,
bool haveParam)
{
Epetra_BlockMap& nonconstUnderlyingMap = const_cast<Epetra_BlockMap&>(uMap);
// Convert underlying map to point map if necessary
Epetra_Map* uPointMapPtr =
dynamic_cast<Epetra_Map*>(&nonconstUnderlyingMap);
bool allocatedPointMap = false;
if (uPointMapPtr == NULL) {
allocatedPointMap = true;
blockMap2PointMap(uMap, uPointMapPtr);
}
int max_gid = uPointMapPtr->MaxAllGID();
int num_global_elements = uPointMapPtr->NumGlobalElements();
int num_my_elements = uPointMapPtr->NumMyElements();
int *global_elements = uPointMapPtr->MyGlobalElements();
const Epetra_Comm& comm = uPointMapPtr->Comm();
int index_base = uPointMapPtr->IndexBase();
int ext_num_global_elements;
int ext_num_my_elements;
int *ext_global_elements;
// Compute number of extended global elements
if (buildImporter)
ext_num_global_elements =
num_global_elements + numConstraints*comm.NumProc();
else
ext_num_global_elements = num_global_elements + numConstraints;
// Compute number of extended local elements
if (buildImporter || haveParam)
ext_num_my_elements = num_my_elements + numConstraints;
else
ext_num_my_elements = num_my_elements;
// Allocate extended global elements array
ext_global_elements = new int[ext_num_my_elements];
// Set extended global elements
for (int i=0; i<num_my_elements; i++) {
ext_global_elements[i] = global_elements[i];
}
if (buildImporter || haveParam)
for (int i=0; i<numConstraints; i++)
ext_global_elements[num_my_elements+i] = max_gid + 1 + i;
// Create extended point map
eMapPtr = new Epetra_Map(ext_num_global_elements, ext_num_my_elements,
ext_global_elements, index_base, comm);
// Free global elements array
delete [] ext_global_elements;
if (allocatedPointMap)
delete uPointMapPtr;
}
int Amesos_TestSolver( Epetra_Comm &Comm, char *matrix_file,
SparseSolverType SparseSolver,
bool transpose,
int special, AMESOS_MatrixType matrix_type ) {
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
std::string FileName = matrix_file ;
int FN_Size = FileName.size() ;
std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size );
std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size );
bool NonContiguousMap = false;
if ( LastFiveBytes == ".triU" ) {
// Call routine to read in unsymmetric Triplet matrix
NonContiguousMap = true;
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, false, Comm, readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFiveBytes == ".triS" ) {
NonContiguousMap = true;
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, true, Comm, readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFourBytes == ".mtx" ) {
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( matrix_file, Comm, readMap,
readA, readx, readb, readxexact) );
} else {
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra( matrix_file, Comm, readMap, readA, readx,
readb, readxexact) ;
}
}
}
Epetra_CrsMatrix transposeA(Copy, *readMap, 0);
Epetra_CrsMatrix *serialA ;
if ( transpose ) {
assert( CrsMatrixTranspose( readA, &transposeA ) == 0 );
serialA = &transposeA ;
} else {
serialA = readA ;
}
Epetra_RowMatrix * passA = 0;
Epetra_Vector * passx = 0;
Epetra_Vector * passb = 0;
Epetra_Vector * passxexact = 0;
Epetra_Vector * passresid = 0;
Epetra_Vector * passtmp = 0;
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
Epetra_Map* map_;
if( NonContiguousMap ) {
//
// map gives us NumMyElements and MyFirstElement;
//
int NumGlobalElements = readMap->NumGlobalElements();
int NumMyElements = map.NumMyElements();
int MyFirstElement = map.MinMyGID();
std::vector<int> MapMap_( NumGlobalElements );
readMap->MyGlobalElements( &MapMap_[0] ) ;
Comm.Broadcast( &MapMap_[0], NumGlobalElements, 0 ) ;
map_ = new Epetra_Map( NumGlobalElements, NumMyElements, &MapMap_[MyFirstElement], 0, Comm);
} else {
map_ = new Epetra_Map( map ) ;
}
Epetra_CrsMatrix A(Copy, *map_, 0);
const Epetra_Map &OriginalMap = serialA->RowMatrixRowMap() ;
assert( OriginalMap.SameAs(*readMap) );
Epetra_Export exporter(OriginalMap, *map_);
Epetra_Export exporter2(OriginalMap, *map_);
Epetra_Export MatrixExporter(OriginalMap, *map_);
Epetra_CrsMatrix AwithDiag(Copy, *map_, 0);
Epetra_Vector x(*map_);
Epetra_Vector b(*map_);
Epetra_Vector xexact(*map_);
Epetra_Vector resid(*map_);
Epetra_Vector readresid(*readMap);
Epetra_Vector tmp(*map_);
Epetra_Vector readtmp(*readMap);
// Epetra_Vector xcomp(*map_); // X as computed by the solver
bool distribute_matrix = ( matrix_type == AMESOS_Distributed ) ;
if ( distribute_matrix ) {
// Create Exporter to distribute read-in matrix and vectors
//
// Initialize x, b and xexact to the values read in from the file
//
x.Export(*readx, exporter, Add);
b.Export(*readb, exporter, Add);
xexact.Export(*readxexact, exporter, Add);
Comm.Barrier();
A.Export(*serialA, exporter, Add);
assert(A.FillComplete()==0);
Comm.Barrier();
passA = &A;
passx = &x;
passb = &b;
passxexact = &xexact;
passresid = &resid;
passtmp = &tmp;
} else {
passA = serialA;
passx = readx;
passb = readb;
passxexact = readxexact;
passresid = &readresid;
passtmp = &readtmp;
}
Epetra_MultiVector CopyB( *passb ) ;
double Anorm = passA->NormInf() ;
SparseDirectTimingVars::SS_Result.Set_Anorm(Anorm) ;
Epetra_LinearProblem Problem( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb );
for ( int i = 0; i < 1+special ; i++ ) {
Epetra_Time TotalTime( Comm ) ;
if ( false ) {
// TEST_UMFPACK is never set by configure
#ifdef HAVE_AMESOS_SUPERLUDIST
} else if ( SparseSolver == SUPERLUDIST ) {
Teuchos::ParameterList ParamList ;
ParamList.set( "MaxProcs", -3 );
Amesos_Superludist A_Superludist( Problem ) ;
//ParamList.set( "Redistribute", true );
//ParamList.set( "AddZeroToDiag", true );
Teuchos::ParameterList& SuperludistParams = ParamList.sublist("Superludist") ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_Superludist.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_Superludist.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_Superludist.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_Superludist.NumericFactorization( ) );
EPETRA_CHK_ERR( A_Superludist.Solve( ) );
#endif
#ifdef HAVE_AMESOS_DSCPACK
} else if ( SparseSolver == DSCPACK ) {
Teuchos::ParameterList ParamList ;
ParamList.set( "MaxProcs", -3 );
Amesos_Dscpack A_dscpack( Problem ) ;
EPETRA_CHK_ERR( A_dscpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_dscpack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_dscpack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_dscpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SCALAPACK
} else if ( SparseSolver == SCALAPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Scalapack A_scalapack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_scalapack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_scalapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_scalapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_scalapack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_scalapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_TAUCS
} else if ( SparseSolver == TAUCS ) {
Teuchos::ParameterList ParamList ;
Amesos_Taucs A_taucs( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_taucs.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_taucs.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_taucs.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_taucs.NumericFactorization( ) );
EPETRA_CHK_ERR( A_taucs.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARDISO
} else if ( SparseSolver == PARDISO ) {
Teuchos::ParameterList ParamList ;
Amesos_Pardiso A_pardiso( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_pardiso.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_pardiso.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_pardiso.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_pardiso.NumericFactorization( ) );
EPETRA_CHK_ERR( A_pardiso.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARAKLETE
} else if ( SparseSolver == PARAKLETE ) {
Teuchos::ParameterList ParamList ;
Amesos_Paraklete A_paraklete( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_paraklete.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_paraklete.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_paraklete.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_paraklete.NumericFactorization( ) );
EPETRA_CHK_ERR( A_paraklete.Solve( ) );
#endif
#ifdef HAVE_AMESOS_MUMPS
} else if ( SparseSolver == MUMPS ) {
Teuchos::ParameterList ParamList ;
Amesos_Mumps A_mumps( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_mumps.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_mumps.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_mumps.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_mumps.NumericFactorization( ) );
EPETRA_CHK_ERR( A_mumps.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SUPERLU
} else if ( SparseSolver == SUPERLU ) {
Teuchos::ParameterList ParamList ;
Amesos_Superlu A_superlu( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_superlu.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_superlu.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_superlu.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_superlu.NumericFactorization( ) );
EPETRA_CHK_ERR( A_superlu.Solve( ) );
#endif
#ifdef HAVE_AMESOS_LAPACK
} else if ( SparseSolver == LAPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Lapack A_lapack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_lapack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_lapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_lapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_lapack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_lapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_UMFPACK
} else if ( SparseSolver == UMFPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Umfpack A_umfpack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_umfpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_umfpack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_umfpack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_umfpack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_umfpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_KLU
} else if ( SparseSolver == KLU ) {
using namespace Teuchos;
Amesos_Time AT;
int setupTimePtr = -1, symTimePtr = -1, numTimePtr = -1, refacTimePtr = -1, solveTimePtr = -1;
AT.CreateTimer(Comm, 2);
AT.ResetTimer(0);
Teuchos::ParameterList ParamList ;
// ParamList.set("OutputLevel",2);
Amesos_Klu A_klu( Problem );
ParamList.set( "MaxProcs", -3 );
ParamList.set( "TrustMe", false );
// ParamList.set( "Refactorize", true );
EPETRA_CHK_ERR( A_klu.SetParameters( ParamList ) ) ;
EPETRA_CHK_ERR( A_klu.SetUseTranspose( transpose ) );
setupTimePtr = AT.AddTime("Setup", setupTimePtr, 0);
EPETRA_CHK_ERR( A_klu.SymbolicFactorization( ) );
symTimePtr = AT.AddTime("Symbolic", symTimePtr, 0);
EPETRA_CHK_ERR( A_klu.NumericFactorization( ) );
numTimePtr = AT.AddTime("Numeric", numTimePtr, 0);
EPETRA_CHK_ERR( A_klu.NumericFactorization( ) );
refacTimePtr = AT.AddTime("Refactor", refacTimePtr, 0);
// for ( int i=0; i<100000 ; i++ )
EPETRA_CHK_ERR( A_klu.Solve( ) );
solveTimePtr = AT.AddTime("Solve", solveTimePtr, 0);
double SetupTime = AT.GetTime(setupTimePtr);
double SymbolicTime = AT.GetTime(symTimePtr);
double NumericTime = AT.GetTime(numTimePtr);
double RefactorTime = AT.GetTime(refacTimePtr);
double SolveTime = AT.GetTime(solveTimePtr);
std::cout << __FILE__ << "::" << __LINE__ << " SetupTime = " << SetupTime << std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " SymbolicTime = " << SymbolicTime - SetupTime << std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " NumericTime = " << NumericTime - SymbolicTime<< std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " RefactorTime = " << RefactorTime - NumericTime << std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " SolveTime = " << SolveTime - RefactorTime << std::endl ;
#endif
} else {
SparseDirectTimingVars::log_file << "Solver not implemented yet" << std::endl ;
std::cerr << "\n\n#################### Requested solver not available on this platform ##################### ATS\n" << std::endl ;
std::cout << " SparseSolver = " << SparseSolver << std::endl ;
std::cerr << " SparseSolver = " << SparseSolver << std::endl ;
}
SparseDirectTimingVars::SS_Result.Set_Total_Time( TotalTime.ElapsedTime() );
} // end for (int i=0; i<special; i++ )
//
// Compute the error = norm(xcomp - xexact )
//
double error;
passresid->Update(1.0, *passx, -1.0, *passxexact, 0.0);
passresid->Norm2(&error);
SparseDirectTimingVars::SS_Result.Set_Error(error) ;
// passxexact->Norm2(&error ) ;
// passx->Norm2(&error ) ;
//
// Compute the residual = norm(Ax - b)
//
double residual ;
passA->Multiply( transpose, *passx, *passtmp);
passresid->Update(1.0, *passtmp, -1.0, *passb, 0.0);
// passresid->Update(1.0, *passtmp, -1.0, CopyB, 0.0);
passresid->Norm2(&residual);
SparseDirectTimingVars::SS_Result.Set_Residual(residual) ;
double bnorm;
passb->Norm2( &bnorm ) ;
SparseDirectTimingVars::SS_Result.Set_Bnorm(bnorm) ;
double xnorm;
passx->Norm2( &xnorm ) ;
SparseDirectTimingVars::SS_Result.Set_Xnorm(xnorm) ;
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
delete map_;
Comm.Barrier();
return 0;
}
/* Find the DBBD form */
int shylu_symbolic_factor
(
Epetra_CrsMatrix *A, // i/p: A matrix
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure, TODO: Required ?
shylu_config *config // i/p: library configuration
)
{
#ifdef TIMING_OUTPUT
Teuchos::Time symtime("symbolic time");
symtime.start();
#endif
int myPID = A->Comm().MyPID();
int n = A->NumGlobalRows();
int Dnr;
int Snr;
int *DRowElems;
int *SRowElems;
int sym = config->sym;
checkMaps(A);
// Get column map
Epetra_Map AColMap = A->ColMap();
int ncols = AColMap.NumMyElements();
int *cols = AColMap.MyGlobalElements();
// Get row map
Epetra_Map ARowMap = A->RowMap();
int nrows = ARowMap.NumMyElements();
int *rows = ARowMap.MyGlobalElements();
// Find all columns in this proc
int *gvals = new int[n]; // vector of size n, not ncols !
// gvals[local cols] = 1, gvals[shared cols] > 1.
int SNumGlobalCols;
findLocalColumns(A, gvals, SNumGlobalCols);
// See if you can shrink the separator by assigning more rows/columns to
// the block diagonals
// TODO: This is because of a bug in coloring remove the if once that is
// fixed
//if (config->schurApproxMethod == 2)
if (config->sep_type == 2)
findNarrowSeparator(A, gvals);
// 3. Assemble diagonal block and the border in convenient form [
/* In each processor, we have (in a permuted form)
* | D_i C_i |
* | R_i S_i |
* D_i - diagonal block, C_i - Column Separator, R_i - Row separator
* S_i - A22 block corresponding to Schur complement part of A
* Assemble all four blocks in local matrices. */
ostringstream ssmsg1;
ssmsg1 << "PID =" << myPID << " ";
string msg = ssmsg1.str();
ssmsg1.clear(); ssmsg1.str("");
// Find #cols in each block
int Dnc = 0; // #cols in diagonal block
int Snc = 0; // #cols in the col. separator
/* Looping on cols will work only for wide separator
* as for narrow sep there will be some sep cols with gvals[col] ==1
* */
/*for (int i = 0; i < ncols ; i++)
{
if (gvals[cols[i]] == 1)
Dnc++;
else
Snc++;
}
// Find #rows in each block
Dnr = Dnc; // #rows in square diagonal block
Snr = nrows - Dnr; // #rows in the row separator*/
// Find #rows in each block
Dnr = 0;
Snr = 0;
for (int i = 0; i < nrows ; i++)
{
if (gvals[rows[i]] == 1)
Dnr++;
else
Snr++;
}
Dnc = Dnr;
// TODO: Snc is no longer useful, should remove it
for (int i = 0; i < ncols ; i++)
{
if (gvals[cols[i]] != 1)
Snc++;
}
assert(Snc >= 0);
// TODO : The above assignment may not be correct in the unsymetric case
////config->dm.print(2, msg + " Mycols=");
cout << msg << " Mycols="<< ncols << "Myrows ="<< nrows << endl;
cout << msg << " #rows and #cols in diagonal blk ="<< Dnr << endl;
cout << msg << " #columns in S ="<< Snc << endl;
cout << msg << " #rows in S ="<< Snr << endl;
ostringstream pidstr;
pidstr << myPID ;
// Create a row map for the D and S blocks [
DRowElems = new int[Dnr];
SRowElems = new int[Snr];
int gid;
// Assemble row ids in two arrays (for D and R blocks)
if (sym)
{
findBlockElems(A, nrows, rows, gvals, Dnr, DRowElems, Snr, SRowElems,
"D"+pidstr.str()+"Rows", "S"+pidstr.str()+"Rows", false) ;
}
else
{
// SRowElems are not known until factorization, TODO
assert(0 == 1);
}
data->Dnr = Dnr;
data->Snr = Snr;
data->Dnc = Dnc;
data->DRowElems = DRowElems;
data->SRowElems = SRowElems;
// Create a column map for the D and S blocks [
int *DColElems = new int[Dnc]; // Elems in column map of D
int *SColElems = new int[Snc]; // Elems in column map of C TODO: Unused
// Assemble column ids in two arrays (for D and C blocks)
findBlockElems(A, ncols, cols, gvals, Dnc, DColElems, Snc, SColElems,
"D"+pidstr.str()+"Cols", "S"+pidstr.str()+"Cols", true) ;
data->DColElems = DColElems;
data->gvals = gvals;
for (int i = 0; i < Snr; i++)
{
// Epetra guarentees columns corresponding to local rows will be first
// in the column map.
assert(SRowElems[i] == SColElems[i]);
}
// ]
/*--Create the Epetra Matrices with the maps (does not insert values) --- */
create_matrices(A, ssym, data, config);
/*--Extract the Epetra Matrices and call fillComplete --- */
extract_matrices(A, ssym, data, config, true);
delete[] SColElems;
Amesos Factory;
const char* SolverType = config->diagonalBlockSolver.c_str();
bool IsAvailable = Factory.Query(SolverType);
assert(IsAvailable == true);
Teuchos::RCP<Epetra_LinearProblem> LP = Teuchos::RCP<Epetra_LinearProblem>
(new Epetra_LinearProblem());
LP->SetOperator((ssym->D).getRawPtr());
//LP->SetOperator((ssym->DT).getRawPtr()); // for transpose
// Create temp vectors
ssym->Dlhs = Teuchos::RCP<Epetra_MultiVector>
(new Epetra_MultiVector(ssym->D->RowMap(), 16));
ssym->Drhs = Teuchos::RCP<Epetra_MultiVector>
(new Epetra_MultiVector(ssym->D->RowMap(), 16));
ssym->Gvec = Teuchos::RCP<Epetra_MultiVector>
(new Epetra_MultiVector(ssym->G->RowMap(), 16));
LP->SetRHS(ssym->Drhs.getRawPtr());
LP->SetLHS(ssym->Dlhs.getRawPtr());
ssym->ReIdx_LP = Teuchos::RCP<
EpetraExt::ViewTransform<Epetra_LinearProblem> >
(new EpetraExt::LinearProblem_Reindex2(0));
ssym->LP = Teuchos::RCP<Epetra_LinearProblem>(&((*(ssym->ReIdx_LP))(*LP)),
false);
Teuchos::RCP<Amesos_BaseSolver> Solver = Teuchos::RCP<Amesos_BaseSolver>
(Factory.Create(SolverType, *(ssym->LP)));
//config->dm.print(5, "Created the diagonal solver");
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
//Solver->SetUseTranspose(true); // for transpose
Teuchos::ParameterList aList;
aList.set("TrustMe", true);
Solver->SetParameters(aList);
Solver->SymbolicFactorization();
//config->dm.print(3, "Symbolic Factorization done");
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Symbolic Factorization Time" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
ssym->OrigLP = LP;
//ssym->LP = LP;
ssym->Solver = Solver;
if (config->schurApproxMethod == 1)
{
Teuchos::ParameterList pList;
Teuchos::RCP<Isorropia::Epetra::Prober> prober =
Teuchos::RCP<Isorropia::Epetra::Prober> (new
Isorropia::Epetra::Prober((ssym->Sg).getRawPtr(),
pList, false));
//config->dm.print(3, "Doing Coloring");
#ifdef TIMING_OUTPUT
ftime.start();
#endif
prober->color();
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time to color" << ftime.totalElapsedTime() << endl;
ftime.reset();
ftime.start();
#endif
ssym->prober = prober;
}
#ifdef TIMING_OUTPUT
symtime.stop();
cout << "Symbolic Time" << symtime.totalElapsedTime() << endl;
symtime.reset();
#endif
}
int checkmap(Epetra_Map & Map, int NumGlobalElements, int NumMyElements,
int *MyGlobalElements, int IndexBase, Epetra_Comm& Comm,
bool DistributedGlobal)
{
int i, ierr=0, forierr = 0;
EPETRA_TEST_ERR(!Map.ConstantElementSize(),ierr);
EPETRA_TEST_ERR(DistributedGlobal!=Map.DistributedGlobal(),ierr);
EPETRA_TEST_ERR(Map.ElementSize()!=1,ierr);
int *MyElementSizeList = new int[NumMyElements];
EPETRA_TEST_ERR(Map.ElementSizeList(MyElementSizeList)!=0,ierr);
forierr = 0;
for (i=0; i<NumMyElements; i++) forierr += MyElementSizeList[i]!=1;
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyElementSizeList;
const Epetra_Comm & Comm1 = Map.Comm();
EPETRA_TEST_ERR(Comm1.NumProc()!=Comm.NumProc(),ierr);
EPETRA_TEST_ERR(Comm1.MyPID()!=Comm.MyPID(),ierr);
EPETRA_TEST_ERR(Map.IndexBase()!=IndexBase,ierr);
EPETRA_TEST_ERR(!Map.LinearMap() && MyGlobalElements==0,ierr);
EPETRA_TEST_ERR(Map.LinearMap() && MyGlobalElements!=0,ierr);
EPETRA_TEST_ERR(Map.MaxAllGID()!=NumGlobalElements-1+IndexBase,ierr);
EPETRA_TEST_ERR(Map.MaxElementSize()!=1,ierr);
int MaxLID = Map.MaxLID();
EPETRA_TEST_ERR(MaxLID!=NumMyElements-1,ierr);
int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase;
if (Comm.MyPID()>2) MaxMyGID+=3;
if (!DistributedGlobal) MaxMyGID = NumMyElements-1+IndexBase;
EPETRA_TEST_ERR(Map.MaxMyGID()!=MaxMyGID,ierr);
EPETRA_TEST_ERR(Map.MinAllGID()!=IndexBase,ierr);
EPETRA_TEST_ERR(Map.MinElementSize()!=1,ierr);
EPETRA_TEST_ERR(Map.MinLID()!=0,ierr);
int MinMyGID = Comm.MyPID()*NumMyElements+IndexBase;
if (Comm.MyPID()>2) MinMyGID+=3;
if (!DistributedGlobal) MinMyGID = 0;
EPETRA_TEST_ERR(Map.MinMyGID()!=MinMyGID,ierr);
int * MyGlobalElements1 = new int[NumMyElements];
EPETRA_TEST_ERR(Map.MyGlobalElements(MyGlobalElements1)!=0,ierr);
forierr = 0;
if (MyGlobalElements==0)
{
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements1[i]!=MinMyGID+i;
EPETRA_TEST_ERR(forierr,ierr);
}
else {
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements[i]!=MyGlobalElements1[i];
EPETRA_TEST_ERR(forierr,ierr);
}
EPETRA_TEST_ERR(Map.NumGlobalElements()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumGlobalPoints()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumMyElements()!=NumMyElements,ierr);
EPETRA_TEST_ERR(Map.NumMyPoints()!=NumMyElements,ierr);
int MaxMyGID2 = Map.GID(Map.LID(MaxMyGID));
EPETRA_TEST_ERR(MaxMyGID2 != MaxMyGID,ierr);
int MaxLID2 = Map.LID(Map.GID(MaxLID));
EPETRA_TEST_ERR(MaxLID2 != MaxLID,ierr);
EPETRA_TEST_ERR(Map.GID(MaxLID+1) != IndexBase-1,ierr);// MaxLID+1 doesn't exist
EPETRA_TEST_ERR(Map.LID(MaxMyGID+1) != -1,ierr);// MaxMyGID+1 doesn't exist or is on a different processor
EPETRA_TEST_ERR(!Map.MyGID(MaxMyGID),ierr);
EPETRA_TEST_ERR(Map.MyGID(MaxMyGID+1),ierr);
EPETRA_TEST_ERR(!Map.MyLID(MaxLID),ierr);
EPETRA_TEST_ERR(Map.MyLID(MaxLID+1),ierr);
EPETRA_TEST_ERR(!Map.MyGID(Map.GID(MaxLID)),ierr);
EPETRA_TEST_ERR(Map.MyGID(Map.GID(MaxLID+1)),ierr);
EPETRA_TEST_ERR(!Map.MyLID(Map.LID(MaxMyGID)),ierr);
EPETRA_TEST_ERR(Map.MyLID(Map.LID(MaxMyGID+1)),ierr);
// Check RemoteIDList function
// Get some GIDs off of each processor to test
int TotalNumEle, NumElePerProc, NumProc = Comm.NumProc();
int MinNumEleOnProc;
int NumMyEle=Map.NumMyElements();
Comm.MinAll(&NumMyEle,&MinNumEleOnProc,1);
if (MinNumEleOnProc > 5) NumElePerProc = 6;
else NumElePerProc = MinNumEleOnProc;
if (NumElePerProc > 0) {
TotalNumEle = NumElePerProc*NumProc;
int * MyGIDlist = new int[NumElePerProc];
int * GIDlist = new int[TotalNumEle];
int * PIDlist = new int[TotalNumEle];
int * LIDlist = new int[TotalNumEle];
for (i=0; i<NumElePerProc; i++)
MyGIDlist[i] = MyGlobalElements1[i];
Comm.GatherAll(MyGIDlist,GIDlist,NumElePerProc);// Get a few values from each proc
Map.RemoteIDList(TotalNumEle, GIDlist, PIDlist, LIDlist);
int MyPID= Comm.MyPID();
forierr = 0;
for (i=0; i<TotalNumEle; i++) {
if (Map.MyGID(GIDlist[i])) {
forierr += PIDlist[i] != MyPID;
forierr += !Map.MyLID(Map.LID(GIDlist[i])) || Map.LID(GIDlist[i]) != LIDlist[i] || Map.GID(LIDlist[i]) != GIDlist[i];
}
else {
forierr += PIDlist[i] == MyPID; // If MyGID comes back false, the PID listed should be that of another proc
}
}
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyGIDlist;
delete [] GIDlist;
delete [] PIDlist;
delete [] LIDlist;
}
delete [] MyGlobalElements1;
// Check RemoteIDList function (assumes all maps are linear, even if not stored that way)
if (Map.LinearMap()) {
int * GIDList = new int[3];
int * PIDList = new int[3];
int * LIDList = new int[3];
int MyPID = Map.Comm().MyPID();
int NumIDs = 0;
//GIDList[NumIDs++] = Map.MaxAllGID()+1; // Should return -1 for both PID and LID
if (Map.MinMyGID()-1>=Map.MinAllGID()) GIDList[NumIDs++] = Map.MinMyGID()-1;
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) GIDList[NumIDs++] = Map.MaxMyGID()+1;
Map.RemoteIDList(NumIDs, GIDList, PIDList, LIDList);
NumIDs = 0;
//EPETRA_TEST_ERR(!(PIDList[NumIDs]==-1),ierr);
//EPETRA_TEST_ERR(!(LIDList[NumIDs++]==-1),ierr);
if (Map.MinMyGID()-1>=Map.MinAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs++]==MyPID-1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs]==MyPID+1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(LIDList[NumIDs++]==0),ierr);
delete [] GIDList;
delete [] PIDList;
delete [] LIDList;
}
return (ierr);
}
// ============================================================================
void EpetraExt::XMLWriter::
Write(const std::string& Label, const Epetra_Map& Map)
{
TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error,
"No file has been opened");
long long NumGlobalElements = Map.NumGlobalElements64();
const int* MyGlobalElements_int = 0;
const long long* MyGlobalElements_LL = 0;
Map.MyGlobalElements(MyGlobalElements_int, MyGlobalElements_LL);
if(!MyGlobalElements_int || !MyGlobalElements_LL)
throw "EpetraExt::XMLWriter::Write: ERROR, GlobalIndices type unknown.";
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<Map Label=\"" << Label
<< "\" NumElements=\"" << NumGlobalElements << '"'
<< " IndexBase=\"" << Map.IndexBase64() << '"'
<< " NumProc=\"" << Comm_.NumProc() << '"';
of.close();
}
for (int iproc = 0; iproc < Comm_.NumProc(); ++iproc)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << " ElementsOnProc" << iproc << "=\"" << Map.NumMyElements() << '"';
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << '>' << std::endl;
of.close();
}
for (int iproc = 0; iproc < Comm_.NumProc(); iproc++)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<Proc ID=\"" << Comm_.MyPID() << "\">" << std::endl;
if(MyGlobalElements_int)
{
for (int i = 0; i < Map.NumMyElements(); ++i)
{
of << MyGlobalElements_int[i] << std::endl;
}
}
else
{
for (int i = 0; i < Map.NumMyElements(); ++i)
{
of << MyGlobalElements_LL[i] << std::endl;
}
}
of << "</Proc>" << std::endl;
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "</Map>" << std::endl;
of.close();
}
}
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp; //
//
// MPI initialization using Teuchos
//
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
//
// Parameters
//
GlobalOrdinal numGlobalElements = 256; // problem size
//
// Construct the problem
//
// Construct a Map that puts approximately the same number of equations on each processor
const Epetra_Map map(numGlobalElements, 0, comm);
// Get update list and number of local equations from newly created map.
const size_t numMyElements = map.NumMyElements();
const GlobalOrdinal* myGlobalElements = map.MyGlobalElements();
// Create a CrsMatrix using the map, with a dynamic allocation of 3 entries per row
RCP<Epetra_CrsMatrix> A = rcp(new Epetra_CrsMatrix(Copy, map, 3));
// Add rows one-at-a-time
for (size_t i = 0; i < numMyElements; i++) {
if (myGlobalElements[i] == 0) {
//TODO: should be rewritten in an Epetra style
A->InsertGlobalValues(myGlobalElements[i], 2,
Teuchos::tuple<Scalar> (2.0, -1.0).getRawPtr(),
Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i], myGlobalElements[i] +1).getRawPtr());
}
else if (myGlobalElements[i] == numGlobalElements - 1) {
A->InsertGlobalValues(myGlobalElements[i], 2,
Teuchos::tuple<Scalar> (-1.0, 2.0).getRawPtr(),
Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i] -1, myGlobalElements[i]).getRawPtr());
}
else {
A->InsertGlobalValues(myGlobalElements[i], 3,
Teuchos::tuple<Scalar> (-1.0, 2.0, -1.0).getRawPtr(),
Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i] -1, myGlobalElements[i], myGlobalElements[i] +1).getRawPtr());
}
}
// Complete the fill, ask that storage be reallocated and optimized
A->FillComplete();
//
// Construct a multigrid preconditioner
//
// Turns a Epetra_CrsMatrix into a MueLu::Matrix
RCP<Xpetra::CrsMatrix<SC, LO, GO, NO, LMO> > mueluA_ = rcp(new Xpetra::EpetraCrsMatrix(A)); //TODO: should not be needed
RCP<Xpetra::Matrix <SC, LO, GO, NO, LMO> > mueluA = rcp(new Xpetra::CrsMatrixWrap<SC, LO, GO, NO, LMO>(mueluA_));
// Multigrid Hierarchy
RCP<Hierarchy> H = rcp(new Hierarchy(mueluA));
H->setVerbLevel(Teuchos::VERB_HIGH);
// Multigrid setup phase (using default parameters)
H->Setup();
//
// Define RHS / LHS
//
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(map));
RCP<Epetra_Vector> B = rcp(new Epetra_Vector(map));
X->PutScalar((Scalar) 0.0);
B->SetSeed(846930886); B->Random();
#ifndef HAVE_MUELU_BELOS
//
// Use AMG directly as an iterative solver (not as a preconditionner)
//
int nIts = 9;
// Wrap Epetra Vectors into Xpetra Vectors
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(X));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(B));
H->Iterate(*mueluB, nIts, *mueluX);
// Print relative residual norm
ST::magnitudeType residualNorms = Utils::ResidualNorm(*mueluA, *mueluX, *mueluB)[0];
if (comm.MyPID() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
#else // HAVE_MUELU_BELOS
//
// Solve Ax = b using AMG as a preconditioner in Belos
//
// Matrix and Multivector type that will be used with Belos
typedef Epetra_MultiVector MV;
typedef Belos::OperatorT<MV> OP;
// Define Operator and Preconditioner
RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(mueluA)); // Turns a Xpetra::Matrix object into a Belos operator
RCP<OP> belosPrec = rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 20;
double tol = 1e-4;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = solver->solve();
// Get the number of iterations for this solve.
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
// Compute actual residuals.
int numrhs=1;
bool badRes = false;
std::vector<SC> actual_resids(numrhs);
std::vector<SC> rhs_norm(numrhs);
RCP<Epetra_MultiVector > resid = rcp(new Epetra_MultiVector(map, numrhs));
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply(*belosOp, *X, *resid);
MVT::MvAddMv(-1.0, *resid, 1.0, *B, *resid);
MVT::MvNorm(*resid, actual_resids);
MVT::MvNorm(*B, rhs_norm);
std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for (int i = 0; i < numrhs; i++) {
SC actRes = actual_resids[i]/rhs_norm[i];
std::cout <<"Problem " << i << " : \t" << actRes <<std::endl;
if (actRes > tol) { badRes = true; }
}
// Check convergence
if (ret != Belos::Converged || badRes) {
std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
return EXIT_FAILURE;
}
std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif // HAVE_MUELU_BELOS
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return EXIT_SUCCESS;
}
int
main (int argc, char *argv[])
{
// These "using" statements make the code a bit more concise.
using std::cout;
using std::endl;
int ierr = 0, i;
// If Trilinos was built with MPI, initialize MPI, otherwise
// initialize the serial "communicator" that stands in for MPI.
#ifdef EPETRA_MPI
MPI_Init (&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
const int MyPID = Comm.MyPID();
const int NumProc = Comm.NumProc();
// We only allow (MPI) Process 0 to write to stdout.
const bool verbose = (MyPID == 0);
const int NumGlobalElements = 100;
if (verbose)
cout << Epetra_Version() << endl << endl;
// Asking the Epetra_Comm to print itself is a good test for whether
// you are running in an MPI environment. However, it will print
// something on all MPI processes, so you should remove it for a
// large-scale parallel run.
cout << Comm << endl;
if (NumGlobalElements < NumProc)
{
if (verbose)
cout << "numGlobalBlocks = " << NumGlobalElements
<< " cannot be < number of processors = " << NumProc << endl;
std::exit (EXIT_FAILURE);
}
// Construct a Map that puts approximately the same number of rows
// of the matrix A on each processor.
Epetra_Map Map (NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map.MyGlobalElements(&MyGlobalElements[0]);
// NumNz[i] is the number of nonzero elements in row i of the sparse
// matrix on this MPI process. Epetra_CrsMatrix uses this to figure
// out how much space to allocate.
std::vector<int> NumNz (NumMyElements);
// We are building a tridiagonal matrix where each row contains the
// nonzero elements (-1 2 -1). Thus, we need 2 off-diagonal terms,
// except for the first and last row of the matrix.
for (int i = 0; i < NumMyElements; ++i)
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1)
NumNz[i] = 2; // First or last row
else
NumNz[i] = 3; // Not the (first or last row)
// Create the Epetra_CrsMatrix.
Epetra_CrsMatrix A (Copy, Map, &NumNz[0]);
//
// Add rows to the sparse matrix one at a time.
//
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
const double two = 2.0;
int NumEntries;
for (int i = 0; i < NumMyElements; ++i)
{
if (MyGlobalElements[i] == 0)
{ // The first row of the matrix.
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == NumGlobalElements - 1)
{ // The last row of the matrix.
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
}
else
{ // Any row of the matrix other than the first or last.
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
ierr = A.InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert (ierr==0);
// Insert the diagonal entry.
ierr = A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i]);
assert(ierr==0);
}
// Finish up. We can call FillComplete() with no arguments, because
// the matrix is square.
ierr = A.FillComplete ();
assert (ierr==0);
// Parameters for the power method.
const int niters = NumGlobalElements*10;
const double tolerance = 1.0e-2;
//
// Run the power method. Keep track of the flop count and the total
// elapsed time.
//
Epetra_Flops counter;
A.SetFlopCounter(counter);
Epetra_Time timer(Comm);
double lambda = 0.0;
ierr += powerMethod (lambda, A, niters, tolerance, verbose);
double elapsedTime = timer.ElapsedTime ();
double totalFlops =counter.Flops ();
// Mflop/s: Million floating-point arithmetic operations per second.
double Mflop_per_s = totalFlops / elapsedTime / 1000000.0;
if (verbose)
cout << endl << endl << "Total Mflop/s for first solve = "
<< Mflop_per_s << endl<< endl;
// Increase the first (0,0) diagonal entry of the matrix.
if (verbose)
cout << endl << "Increasing magnitude of first diagonal term, solving again"
<< endl << endl << endl;
if (A.MyGlobalRow (0)) {
int numvals = A.NumGlobalEntries (0);
std::vector<double> Rowvals (numvals);
std::vector<int> Rowinds (numvals);
A.ExtractGlobalRowCopy (0, numvals, numvals, &Rowvals[0], &Rowinds[0]); // Get A(0,0)
for (int i = 0; i < numvals; ++i)
if (Rowinds[i] == 0)
Rowvals[i] *= 10.0;
A.ReplaceGlobalValues (0, numvals, &Rowvals[0], &Rowinds[0]);
}
//
// Run the power method again. Keep track of the flop count and the
// total elapsed time.
//
lambda = 0.0;
timer.ResetStartTime();
counter.ResetFlops();
ierr += powerMethod (lambda, A, niters, tolerance, verbose);
elapsedTime = timer.ElapsedTime();
totalFlops = counter.Flops();
Mflop_per_s = totalFlops / elapsedTime / 1000000.0;
if (verbose)
cout << endl << endl << "Total Mflop/s for second solve = "
<< Mflop_per_s << endl << endl;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return ierr;
}
//
// Diagonal: 0=no change, 1=eliminate entry
// from the map for the largest row element in process 0
// 2=add diagonal entries to the matrix, with a zero value
// (assume row map contains all diagonal entries).
//
// ReindexRowMap:
// 0=no change, 1= add 2 (still contiguous), 2=non-contiguous
//
// ReindexColMap
// 0=same as RowMap, 1=add 4 - Different From RowMap, but contiguous)
//
// RangeMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
// DomainMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
RCP<Epetra_CrsMatrix> NewMatNewMap(Epetra_CrsMatrix& In,
int Diagonal,
int ReindexRowMap,
int ReindexColMap,
int RangeMapType,
int DomainMapType
)
{
//
// If we are making no change, return the original matrix (which has a linear map)
//
#if 0
std::cout << __FILE__ << "::" << __LINE__ << " "
<< Diagonal << " "
<< ReindexRowMap << " "
<< ReindexColMap << " "
<< RangeMapType << " "
<< DomainMapType << " " << std::endl ;
#endif
if ( Diagonal + ReindexRowMap + ReindexColMap + RangeMapType + DomainMapType == 0 ) {
RCP<Epetra_CrsMatrix> ReturnOrig = rcp( &In, false );
return ReturnOrig ;
}
//
// Diagonal==2 is used for a different purpose -
// Making sure that the diagonal of the matrix is non-empty.
// Note: The diagonal must exist in In.RowMap().
//
if ( Diagonal == 2 ) {
assert( ReindexRowMap==0 && ReindexColMap == 0 ) ;
}
int (*RowPermute)(int in) = 0;
int (*ColPermute)(int in) = 0;
assert( Diagonal >= 0 && Diagonal <= 2 );
assert( ReindexRowMap>=0 && ReindexRowMap<=2 );
assert( ReindexColMap>=0 && ReindexColMap<=1 );
assert( RangeMapType>=0 && RangeMapType<=3 );
assert( DomainMapType>=0 && DomainMapType<=3 );
Epetra_Map DomainMap = In.DomainMap();
Epetra_Map RangeMap = In.RangeMap();
Epetra_Map ColMap = In.ColMap();
Epetra_Map RowMap = In.RowMap();
int NumMyRowElements = RowMap.NumMyElements();
int NumMyColElements = ColMap.NumMyElements();
int NumMyRangeElements = RangeMap.NumMyElements();
int NumMyDomainElements = DomainMap.NumMyElements();
int NumGlobalRowElements = RowMap.NumGlobalElements();
int NumGlobalColElements = ColMap.NumGlobalElements();
int NumGlobalRangeElements = RangeMap.NumGlobalElements();
int NumGlobalDomainElements = DomainMap.NumGlobalElements();
assert( NumGlobalRangeElements == NumGlobalDomainElements ) ;
std::vector<int> MyGlobalRowElements( NumMyRowElements ) ;
std::vector<int> NumEntriesPerRow( NumMyRowElements ) ;
std::vector<int> MyPermutedGlobalRowElements( NumMyRowElements ) ;
std::vector<int> MyGlobalColElements( NumMyColElements ) ;
std::vector<int> MyPermutedGlobalColElements( NumMyColElements ) ; // Used to create the column map
std::vector<int> MyPermutedGlobalColElementTable( NumMyColElements ) ; // To convert local indices to global
std::vector<int> MyGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyPermutedGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyGlobalDomainElements( NumMyDomainElements ) ;
std::vector<int> MyPermutedGlobalDomainElements( NumMyDomainElements ) ;
RowMap.MyGlobalElements(&MyGlobalRowElements[0]);
ColMap.MyGlobalElements(&MyGlobalColElements[0]);
RangeMap.MyGlobalElements(&MyGlobalRangeElements[0]);
DomainMap.MyGlobalElements(&MyGlobalDomainElements[0]);
switch( ReindexRowMap ) {
case 0:
RowPermute = &NoPermute ;
break;
case 1:
RowPermute = &SmallRowPermute ;
break;
case 2:
RowPermute = BigRowPermute ;
break;
}
switch( ReindexColMap ) {
case 0:
ColPermute = RowPermute ;
break;
case 1:
ColPermute = &SmallColPermute ;
break;
}
//
// Create Serial Range and Domain Maps based on the permuted indexing
//
int nlocal = 0;
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements;
std::vector<int> AllIDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDs[i] = (*RowPermute)( i ) ;
Epetra_Map SerialRangeMap( -1, nlocal, &AllIDs[0], 0, In.Comm());
std::vector<int> AllIDBs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDBs[i] = (*ColPermute)( i ) ;
Epetra_Map SerialDomainMap( -1, nlocal, &AllIDBs[0], 0, In.Comm());
//
// Create Bizarre Range and Domain Maps based on the permuted indexing
// These are nearly serial, having all but one element on process 0
// The goal here is to make sure that we can use Domain and Range maps
// that are neither serial, nor distributed in the normal manner.
//
std::vector<int> AllIDCs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDCs[i] = (*ColPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDCs[0] = (*ColPermute)( NumGlobalRangeElements - 1 );
}
}
int iam = In.Comm().MyPID();
Epetra_Map BizarreDomainMap( -1, nlocal, &AllIDCs[0], 0, In.Comm());
std::vector<int> AllIDDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDDs[i] = (*RowPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDDs[0] = (*RowPermute)( NumGlobalRangeElements -1 ) ;
}
}
Epetra_Map BizarreRangeMap( -1, nlocal, &AllIDDs[0], 0, In.Comm());
//
// Compute the column map
//
// If Diagonal==1, remove the column corresponding to the last row owned
// by process 0. Removing this column from a tridiagonal matrix, leaves
// a disconnected, but non-singular matrix.
//
int NumMyColElementsOut = 0 ;
int NumGlobalColElementsOut ;
if ( Diagonal == 1 )
NumGlobalColElementsOut = NumGlobalColElements-1;
else
NumGlobalColElementsOut = NumGlobalColElements;
if ( Diagonal == 1 && iam==0 ) {
for ( int i=0; i < NumMyColElements ; i++ ) {
if ( MyGlobalColElements[i] != MyGlobalRowElements[NumMyRowElements-1] ) {
MyPermutedGlobalColElements[NumMyColElementsOut++] =
(*ColPermute)( MyGlobalColElements[i] ) ;
}
}
assert( NumMyColElementsOut == NumMyColElements-1 );
} else {
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElements[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
NumMyColElementsOut = NumMyColElements ;
if ( Diagonal == 2 ) {
// For each row, make sure that the column map has this row in it,
// if it doesn't, add it to the column map.
// Note: MyPermutedGlobalColElements == MyGlobalColElements when
// Diagonal==2 because ( Diagonal == 2 ) implies:
// ReindexRowMap==0 && ReindexColMap == 0 - see assert above
for ( int i=0; i < NumMyRowElements ; i++ ) {
bool MissingDiagonal = true;
for ( int j=0; j < NumMyColElements; j++ ) {
if ( MyGlobalRowElements[i] == MyGlobalColElements[j] ) {
MissingDiagonal = false;
}
}
if ( MissingDiagonal ) {
MyPermutedGlobalColElements.resize(NumMyColElements+1);
MyPermutedGlobalColElements[NumMyColElementsOut] = MyGlobalRowElements[i];
NumMyColElementsOut++;
}
}
In.Comm().SumAll(&NumMyColElementsOut,&NumGlobalColElementsOut,1);
}
}
//
// These tables are used both as the permutation tables and to create the maps.
//
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElementTable[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
for ( int i=0; i < NumMyRowElements ; i++ )
MyPermutedGlobalRowElements[i] =
(*RowPermute)( MyGlobalRowElements[i] ) ;
for ( int i=0; i < NumMyRangeElements ; i++ )
MyPermutedGlobalRangeElements[i] =
(*RowPermute)( MyGlobalRangeElements[i] ) ;
for ( int i=0; i < NumMyDomainElements ; i++ )
MyPermutedGlobalDomainElements[i] =
(*ColPermute)( MyGlobalDomainElements[i] ) ;
RCP<Epetra_Map> PermutedRowMap =
rcp( new Epetra_Map( NumGlobalRowElements, NumMyRowElements,
&MyPermutedGlobalRowElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedColMap =
rcp( new Epetra_Map( NumGlobalColElementsOut, NumMyColElementsOut,
&MyPermutedGlobalColElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedRangeMap =
rcp( new Epetra_Map( NumGlobalRangeElements, NumMyRangeElements,
&MyPermutedGlobalRangeElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedDomainMap =
rcp( new Epetra_Map( NumGlobalDomainElements, NumMyDomainElements,
&MyPermutedGlobalDomainElements[0], 0, In.Comm() ) );
//
// These vectors are filled and then passed to InsertGlobalValues
//
std::vector<int> ThisRowIndices( In.MaxNumEntries() );
std::vector<double> ThisRowValues( In.MaxNumEntries() );
std::vector<int> PermutedGlobalColIndices( In.MaxNumEntries() );
//std::cout << __FILE__ << "::" <<__LINE__ << std::endl ;
RCP<Epetra_CrsMatrix> Out =
rcp( new Epetra_CrsMatrix( Copy, *PermutedRowMap, *PermutedColMap, 0 ) );
for (int i=0; i<NumMyRowElements; i++)
{
int NumIndicesThisRow = 0;
assert( In.ExtractMyRowCopy( i,
In.MaxNumEntries(),
NumIndicesThisRow,
&ThisRowValues[0],
&ThisRowIndices[0] ) == 0 ) ;
for (int j = 0 ; j < NumIndicesThisRow ; j++ )
{
PermutedGlobalColIndices[j] = MyPermutedGlobalColElementTable[ ThisRowIndices[j] ] ;
}
bool MissingDiagonal = false;
if ( Diagonal==2 ) {
//
assert( MyGlobalRowElements[i] == MyPermutedGlobalRowElements[i] );
MissingDiagonal = true;
for( int j =0 ; j < NumIndicesThisRow ; j++ ) {
if ( PermutedGlobalColIndices[j] == MyPermutedGlobalRowElements[i] ) {
MissingDiagonal = false ;
}
}
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< " MyPermutedGlobalRowElements[i] = " << MyPermutedGlobalRowElements[i]
<< " MissingDiagonal = " << MissingDiagonal << std::endl ;
#endif
}
if ( MissingDiagonal ) {
ThisRowValues.resize(NumIndicesThisRow+1) ;
ThisRowValues[NumIndicesThisRow] = 0.0;
PermutedGlobalColIndices.resize(NumIndicesThisRow+1);
PermutedGlobalColIndices[NumIndicesThisRow] = MyPermutedGlobalRowElements[i] ;
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< "NumIndicesThisRow = " << NumIndicesThisRow
<< "ThisRowValues[NumIndicesThisRow = " << ThisRowValues[NumIndicesThisRow]
<< " PermutedGlobalColIndices[NumIndcesThisRow] = " << PermutedGlobalColIndices[NumIndicesThisRow]
<< std::endl ;
#endif
NumIndicesThisRow++ ;
}
assert( Out->InsertGlobalValues( MyPermutedGlobalRowElements[i],
NumIndicesThisRow,
&ThisRowValues[0],
&PermutedGlobalColIndices[0] ) >= 0 );
}
//
Epetra_LocalMap ReplicatedMap( NumGlobalRangeElements, 0, In.Comm() );
RCP<Epetra_Map> OutRangeMap ;
RCP<Epetra_Map> OutDomainMap ;
switch( RangeMapType ) {
case 0:
OutRangeMap = PermutedRangeMap ;
break;
case 1:
OutRangeMap = rcp(&SerialRangeMap, false);
break;
case 2:
OutRangeMap = rcp(&BizarreRangeMap, false);
break;
case 3:
OutRangeMap = rcp(&ReplicatedMap, false);
break;
}
// switch( DomainMapType ) {
switch( DomainMapType ) {
case 0:
OutDomainMap = PermutedDomainMap ;
break;
case 1:
OutDomainMap = rcp(&SerialDomainMap, false);
break;
case 2:
OutDomainMap = rcp(&BizarreDomainMap, false);
break;
case 3:
OutDomainMap = rcp(&ReplicatedMap, false);
break;
}
#if 0
assert(Out->FillComplete( *PermutedDomainMap, *PermutedRangeMap )==0);
#else
assert(Out->FillComplete( *OutDomainMap, *OutRangeMap )==0);
#endif
#if 0
std::cout << __FILE__ << "::" << __LINE__ << std::endl ;
Out->Print( std::cout ) ;
#endif
return Out;
}
Epetra_CrsMatrix *
Laplace2D::CreateLaplacian( const int nx, const int ny, const Epetra_Comm * Comm)
{
int NumGlobalElements = nx * ny;
// create a map
Epetra_Map * Map = new Epetra_Map(NumGlobalElements,0,*Comm);
// local number of rows
int NumMyElements = Map->NumMyElements();
// get update list
int * MyGlobalElements = Map->MyGlobalElements();
double hx = 1.0/(nx-1);
double hy = 1.0/(ny-1);
double off_left = -1.0/(hx*hx);
double off_right = -1.0/(hx*hx);
double off_lower = -1.0/(hy*hy);
double off_upper = -1.0/(hy*hy);
double diag = 2.0/(hx*hx) + 2.0/(hy*hy);
int left, right, lower, upper;
// a bit overestimated the nonzero per row
Epetra_CrsMatrix * A = new Epetra_CrsMatrix(Copy,*Map,5);
// Add rows one-at-a-time
double * Values = new double[4];
int * Indices = new int[4];
for( int i = 0; i < NumMyElements; ++i )
{
int NumEntries=0;
get_myNeighbours( MyGlobalElements[i], nx, ny, left, right, lower, upper );
if( left != -1 )
{
Indices[NumEntries] = left;
Values[NumEntries] = off_left;
++NumEntries;
}
if( right != -1 )
{
Indices[NumEntries] = right;
Values[NumEntries] = off_right;
++NumEntries;
}
if( lower != -1 )
{
Indices[NumEntries] = lower;
Values[NumEntries] = off_lower;
++NumEntries;
}
if( upper != -1 )
{
Indices[NumEntries] = upper;
Values[NumEntries] = off_upper;
++NumEntries;
}
// put the off-diagonal entries
A->InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices);
// Put in the diagonal entry
A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, MyGlobalElements+i);
}
// put matrix in local ordering
A->FillComplete();
delete [] Indices;
delete [] Values;
delete Map;
return A;
} /* createJacobian */
int TLowCommunicationMakeColMapAndReindex(int N, const int * rowptr, int * colind_LID, const int_type *colind_GID, const Epetra_Map& domainMap, const int * owningPIDs, bool SortGhostsAssociatedWithEachProcessor, std::vector<int>& RemotePIDs, MapType1 & NewColMap)
{
int i,j;
// Sanity checks
bool UseLL;
if(domainMap.GlobalIndicesLongLong()) UseLL=true;
else if(domainMap.GlobalIndicesInt()) UseLL=false;
else throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot detect int type.");
// Scan all column indices and sort into two groups:
// Local: those whose GID matches a GID of the domain map on this processor and
// Remote: All others.
int numDomainElements = domainMap.NumMyElements();
bool * LocalGIDs = 0;
if (numDomainElements>0) LocalGIDs = new bool[numDomainElements];
for (i=0; i<numDomainElements; i++) LocalGIDs[i] = false; // Assume domain GIDs are not local
bool DoSizes = !domainMap.ConstantElementSize(); // If not constant element size, then error
if(DoSizes) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot handle non-constant sized domainMap.");
// In principle it is good to have RemoteGIDs and RemotGIDList be as long as the number of remote GIDs
// on this processor, but this would require two passes through the column IDs, so we make it the max of 100
// and the number of block rows.
const int numMyBlockRows = N;
int hashsize = numMyBlockRows; if (hashsize < 100) hashsize = 100;
Epetra_HashTable<int_type> RemoteGIDs(hashsize);
std::vector<int_type> RemoteGIDList; RemoteGIDList.reserve(hashsize);
std::vector<int> PIDList; PIDList.reserve(hashsize);
// Here we start using the *int* colind array. If int_type==int this clobbers the GIDs, if
// int_type==long long, then this is the first use of the colind array.
// For *local* GID's set colind with with their LID in the domainMap. For *remote* GIDs,
// we set colind with (numDomainElements+NumRemoteColGIDs) before the increment of
// the remote count. These numberings will be separate because no local LID is greater
// than numDomainElements.
int NumLocalColGIDs = 0;
int NumRemoteColGIDs = 0;
for(i = 0; i < numMyBlockRows; i++) {
for(j = rowptr[i]; j < rowptr[i+1]; j++) {
int_type GID = colind_GID[j];
// Check if GID matches a row GID
int LID = domainMap.LID(GID);
if(LID != -1) {
bool alreadyFound = LocalGIDs[LID];
if (!alreadyFound) {
LocalGIDs[LID] = true; // There is a column in the graph associated with this domain map GID
NumLocalColGIDs++;
}
colind_LID[j] = LID;
}
else {
int_type hash_value=RemoteGIDs.Get(GID);
if(hash_value == -1) { // This means its a new remote GID
int PID = owningPIDs[j];
if(PID==-1) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: Cannot figure out if PID is owned.");
colind_LID[j] = numDomainElements + NumRemoteColGIDs;
RemoteGIDs.Add(GID, NumRemoteColGIDs);
RemoteGIDList.push_back(GID);
PIDList.push_back(PID);
NumRemoteColGIDs++;
}
else
colind_LID[j] = numDomainElements + hash_value;
}
}
}
// Possible short-circuit: If all domain map GIDs are present as column indices, then set ColMap=domainMap and quit
if (domainMap.Comm().NumProc()==1) {
if (NumRemoteColGIDs!=0) {
throw std::runtime_error("Some column IDs are not in domainMap. If matrix is rectangular, you must pass in a domainMap");
// Sanity test: When one processor,there can be no remoteGIDs
}
if (NumLocalColGIDs==numDomainElements) {
if (LocalGIDs!=0) delete [] LocalGIDs;
// In this case, we just use the domainMap's indices, which is, not coincidently, what we clobbered colind with up above anyway.
// No further reindexing is needed.
NewColMap = domainMap;
return 0;
}
}
// Now build integer array containing column GIDs
// Build back end, containing remote GIDs, first
int numMyBlockCols = NumLocalColGIDs + NumRemoteColGIDs;
std::vector<int_type> ColIndices;
int_type * RemoteColIndices=0;
if(numMyBlockCols > 0) {
ColIndices.resize(numMyBlockCols);
if(NumLocalColGIDs!=numMyBlockCols) RemoteColIndices = &ColIndices[NumLocalColGIDs]; // Points to back end of ColIndices
else RemoteColIndices=0;
}
for(i = 0; i < NumRemoteColGIDs; i++)
RemoteColIndices[i] = RemoteGIDList[i];
// Build permute array for *remote* reindexing.
std::vector<int> RemotePermuteIDs(NumRemoteColGIDs);
for(i=0; i<NumRemoteColGIDs; i++) RemotePermuteIDs[i]=i;
// Sort External column indices so that all columns coming from a given remote processor are contiguous
int NumListsInt=0;
int NumListsLL =0;
int * IntSortLists[2];
long long * LLSortLists[2];
int * RemotePermuteIDs_ptr = RemotePermuteIDs.size() ? &RemotePermuteIDs[0] : 0;
if(!UseLL) {
// int version
IntSortLists[0] = (int*) RemoteColIndices;
IntSortLists[1] = RemotePermuteIDs_ptr;
NumListsInt=2;
}
else {
//LL version
LLSortLists[0] = (long long*) RemoteColIndices;
IntSortLists[0] = RemotePermuteIDs_ptr;
NumListsInt = NumListsLL = 1;
}
int * PIDList_ptr = PIDList.size() ? &PIDList[0] : 0;
Epetra_Util::Sort(true, NumRemoteColGIDs, PIDList_ptr, 0, 0, NumListsInt, IntSortLists,NumListsLL,LLSortLists);
// Stash the RemotePIDs
PIDList.resize(NumRemoteColGIDs);
RemotePIDs = PIDList;
if (SortGhostsAssociatedWithEachProcessor) {
// Sort external column indices so that columns from a given remote processor are not only contiguous
// but also in ascending order. NOTE: I don't know if the number of externals associated
// with a given remote processor is known at this point ... so I count them here.
// NTS: Only sort the RemoteColIndices this time...
int StartCurrent, StartNext;
StartCurrent = 0; StartNext = 1;
while ( StartNext < NumRemoteColGIDs ) {
if (PIDList[StartNext]==PIDList[StartNext-1]) StartNext++;
else {
IntSortLists[0] = &RemotePermuteIDs[StartCurrent];
Epetra_Util::Sort(true,StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]),0,0,1,IntSortLists,0,0);
StartCurrent = StartNext; StartNext++;
}
}
IntSortLists[0] = &RemotePermuteIDs[StartCurrent];
Epetra_Util::Sort(true, StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]), 0, 0, 1,IntSortLists,0,0);
}
// Reverse the permutation to get the information we actually care about
std::vector<int> ReverseRemotePermuteIDs(NumRemoteColGIDs);
for(i=0; i<NumRemoteColGIDs; i++) ReverseRemotePermuteIDs[RemotePermuteIDs[i]]=i;
// Build permute array for *local* reindexing.
bool use_local_permute=false;
std::vector<int> LocalPermuteIDs(numDomainElements);
// Now fill front end. Two cases:
// (1) If the number of Local column GIDs is the same as the number of Local domain GIDs, we
// can simply read the domain GIDs into the front part of ColIndices, otherwise
// (2) We step through the GIDs of the domainMap, checking to see if each domain GID is a column GID.
// we want to do this to maintain a consistent ordering of GIDs between the columns and the domain.
if(NumLocalColGIDs == domainMap.NumMyElements()) {
if(NumLocalColGIDs > 0) {
domainMap.MyGlobalElements(&ColIndices[0]); // Load Global Indices into first numMyBlockCols elements column GID list
}
}
else {
int_type* MyGlobalElements = 0;
domainMap.MyGlobalElementsPtr(MyGlobalElements);
int* ElementSizeList = 0;
if(DoSizes)
ElementSizeList = domainMap.ElementSizeList();
int NumLocalAgain = 0;
use_local_permute = true;
for(i = 0; i < numDomainElements; i++) {
if(LocalGIDs[i]) {
LocalPermuteIDs[i] = NumLocalAgain;
ColIndices[NumLocalAgain++] = MyGlobalElements[i];
}
}
assert(NumLocalAgain==NumLocalColGIDs); // Sanity test
}
// Done with this array
if (LocalGIDs!=0) delete [] LocalGIDs;
// Make Column map with same element sizes as Domain map
int_type * ColIndices_ptr = ColIndices.size() ? &ColIndices[0] : 0;
MapType2 temp((int_type)(-1), numMyBlockCols, ColIndices_ptr, (int_type)domainMap.IndexBase64(), domainMap.Comm());
NewColMap = temp;
// Low-cost reindex of the matrix
for(i=0; i<numMyBlockRows; i++){
for(j=rowptr[i]; j<rowptr[i+1]; j++){
int ID=colind_LID[j];
if(ID < numDomainElements){
if(use_local_permute) colind_LID[j] = LocalPermuteIDs[colind_LID[j]];
// In the case where use_local_permute==false, we just copy the DomainMap's ordering, which it so happens
// is what we put in colind to begin with.
}
else
colind_LID[j] = NumLocalColGIDs + ReverseRemotePermuteIDs[colind_LID[j]-numDomainElements];
}
}
return 0;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm Comm;
#endif
// My MPI process rank.
const int MyPID = Comm.MyPID();
// "/Users/sakashitatatsuya/Downloads/barista_trunk_slepc/sample/hamiltonian_matrix.ip"
std::ifstream ifs(argv[1]);
alps::Parameters params(ifs);
Teuchos::oblackholestream blackHole;
std::ostream& out = (MyPID == 0) ? std::cout : blackHole;
barista::Hamiltonian<> hamiltonian(params);
matrix_type matrix(hamiltonian.dimension(), hamiltonian.dimension());
hamiltonian.fill<double>(matrix);
int m,n;
int N;
m = n = N = hamiltonian.dimension();
//std::cout << matrix << std::endl;
std::ofstream ofs;
if (MyPID==0) {
ofs.open("anasazi_time.txt");
if (!ofs) {
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
}
//Teuchos::ParameterList GaleriList;
using Teuchos::RCP;
using Teuchos::rcp;
typedef Teuchos::ScalarTraits<double> STS;
const double one = STS::one();
const double zero = STS::zero();
// The problem is defined on a 2D grid, global size is nx * nx.
//int nx = N;
//GaleriList.set("n", nx * nx);
//GaleriList.set("nx", nx);
//GaleriList.set("ny", nx);
//Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( Galeri::CreateMap("Linear", Comm, GaleriList) );
//Teuchos::RCP<Epetra_RowMatrix> A = Teuchos::rcp( Galeri::CreateCrsMatrix("Laplace2D", &*Map, GaleriList) );
// Construct a Map that puts approximately the same number of rows
// of the matrix A on each processor.
Epetra_Map RowMap (N, 0, Comm);
Epetra_Map ColMap (N, 0, Comm);
// Get update list and number of local equations from newly created Map.
const int NumMyRowElements = RowMap.NumMyElements ();
std::vector<int> MyGlobalRowElements (NumMyRowElements);
RowMap.MyGlobalElements (&MyGlobalRowElements[0]);
// Create an Epetra_CrsMatrix using the given row map.
RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Copy, RowMap, n));
// We use info to catch any errors that may have happened during // matrix assembly, and report them globally. We do this so that // the MPI processes won't call FillComplete() unless they all // successfully filled their parts of the matrix.
int info = 0;
try {
//
// Compute coefficients for the discrete integral operator.
//
std::vector<double> Values (n);
std::vector<int> Indices (n);
//const double inv_mp1 = one / (m+1);
//const double inv_np1 = one / (n+1);
int count;
//for (int i = 0; i < n; ++i) {
// Indices[i] = i;
//}
for (int i = 0; i < NumMyRowElements; ++i) {
count =0;
for (int j = 0; j < n; ++j) {
if (matrix(MyGlobalRowElements[i],j)!=0) {
Values[count] = matrix(MyGlobalRowElements[i],j);
Indices[count] = j;
count++;
}
}
info = A->InsertGlobalValues (MyGlobalRowElements[i], count,
&Values[0], &Indices[0]);
// Make sure that the insertion succeeded. Teuchos'
// TEST_FOR_EXCEPTION macro gives a nice error message if the
// thrown exception isn't caught. We'll report this on the
// offending MPI process.
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error, "Failed to insert n="
<< n << " global value" << (n != 1 ? "s" : "")
<< " in row " << MyGlobalRowElements[i]
<< " of the matrix." );
*/
} // for i = 0...
// Call FillComplete on the matrix. Since the matrix isn't square,
// we have to give FillComplete the domain and range maps, which in
// this case are the column resp. row maps.
info = A->FillComplete (ColMap, RowMap);
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error,
"FillComplete failed with INFO = " << info << ".");
*/
info = A->OptimizeStorage();
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error,
"OptimizeStorage failed with INFO = " << info << ".");
*/
} catch (std::runtime_error& e) {
// If multiple MPI processes are reporting errors, sometimes
// forming the error message as a string and then writing it to
// the output stream prevents messages from different processes
// from being interleaved.
std::ostringstream os;
os << "*** Error on MPI process " << MyPID << ": " << e.what();
cerr << os.str() << endl;
if (info == 0)
info = -1; // All procs will share info later on.
}
// Variables used for the Block Davidson Method
const int nev = 5;
const int blockSize = 5;
const int numBlocks = 8;
const int maxRestarts = 500;
const double tol = 1.0e-8;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<double, Epetra_MultiVector> MVT;
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
//
//Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(*Map, blockSize) );
Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(ColMap, blockSize) );
ivec->Random();
// Create the eigenproblem.
Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > problem =
Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(A, ivec) );
// Inform the eigenproblem that the operator A is symmetric
problem->setHermitian(true);
// Set the number of eigenvalues requested
problem->setNEV( nev );
// Inform the eigenproblem that you are finishing passing it information
bool boolret = problem->setProblem();
if (boolret != true) {
std::cout<<"Anasazi::BasicEigenproblem::setProblem() returned an error." << std::endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return -1;
}
// Create parameter list to pass into the solver manager
Teuchos::ParameterList anasaziPL;
anasaziPL.set( "Which", "LM" );
anasaziPL.set( "Block Size", blockSize );
anasaziPL.set( "Maximum Iterations", 500 );
anasaziPL.set( "Convergence Tolerance", tol );
anasaziPL.set( "Verbosity", Anasazi::Errors+Anasazi::Warnings+Anasazi::TimingDetails+Anasazi::FinalSummary );
// Create the solver manager
Anasazi::LOBPCGSolMgr<double, MV, OP> anasaziSolver(problem, anasaziPL);
// Solve the problem
double start, end;
MPI_Barrier(MPI_COMM_WORLD);
start = MPI_Wtime();
Anasazi::ReturnType returnCode = anasaziSolver.solve();
MPI_Barrier(MPI_COMM_WORLD);
end = MPI_Wtime();
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<double,MV> sol = problem->getSolution();
std::vector<Anasazi::Value<double> > evals = sol.Evals;
Teuchos::RCP<MV> evecs = sol.Evecs;
// Compute residuals.
std::vector<double> normR(sol.numVecs);
Teuchos::SerialDenseMatrix<int,double> T(sol.numVecs, sol.numVecs);
Epetra_MultiVector tempAevec( ColMap, sol.numVecs );
T.putScalar(0.0);
for (int i=0; i<sol.numVecs; i++) {
T(i,i) = evals[i].realpart;
}
A->Apply( *evecs, tempAevec );
MVT::MvTimesMatAddMv( -1.0, *evecs, T, 1.0, tempAevec );
MVT::MvNorm( tempAevec, normR );
if (MyPID == 0) {
// Print the results
std::cout<<"Solver manager returned " << (returnCode == Anasazi::Converged ? "converged." : "unconverged.") << std::endl;
std::cout<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
std::cout<<std::setw(16)<<"Eigenvalue"
<<std::setw(18)<<"Direct Residual"
<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
for (int i=0; i<sol.numVecs; i++) {
std::cout<<std::setw(16)<<evals[i].realpart
<<std::setw(18)<<normR[i]/evals[i].realpart
<<std::endl;
}
std::cout<<"------------------------------------------------------"<<std::endl;
}
// Print out the map and matrices
//ColMap.Print (out);
//A->Print (cout);
//RowMap.Print (cout);
double time;
int iter;
if (MyPID==0) {
iter = anasaziSolver.getNumIters();
Teuchos::Array<Teuchos::RCP<Teuchos::Time> > timer = anasaziSolver.getTimers();
Teuchos::RCP<Teuchos::Time> _timerSolve = timer[0];
cout << "timerSolve=" << _timerSolve << endl;
time = end - start;
cout << "time=" << time << endl;
ofs << "time=" << time << endl;
cout << "iter=" << iter << endl;
ofs << "iter=" << iter << endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return 0;
}
//
// Amesos_TestMultiSolver.cpp reads in a matrix in Harwell-Boeing format,
// calls one of the sparse direct solvers, using blocked right hand sides
// and computes the error and residual.
//
// TestSolver ignores the Harwell-Boeing right hand sides, creating
// random right hand sides instead.
//
// Amesos_TestMultiSolver can test either A x = b or A^T x = b.
// This can be a bit confusing because sparse direct solvers
// use compressed column storage - the transpose of Trilinos'
// sparse row storage.
//
// Matrices:
// readA - Serial. As read from the file.
// transposeA - Serial. The transpose of readA.
// serialA - if (transpose) then transposeA else readA
// distributedA - readA distributed to all processes
// passA - if ( distributed ) then distributedA else serialA
//
//
int Amesos_TestMultiSolver( Epetra_Comm &Comm, char *matrix_file, int numsolves,
SparseSolverType SparseSolver, bool transpose,
int special, AMESOS_MatrixType matrix_type ) {
int iam = Comm.MyPID() ;
// int hatever;
// if ( iam == 0 ) std::cin >> hatever ;
Comm.Barrier();
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
std::string FileName = matrix_file ;
int FN_Size = FileName.size() ;
std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size );
std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size );
bool NonContiguousMap = false;
if ( LastFiveBytes == ".triU" ) {
NonContiguousMap = true;
// Call routine to read in unsymmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, false, Comm, readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFiveBytes == ".triS" ) {
NonContiguousMap = true;
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, true, Comm,
readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFourBytes == ".mtx" ) {
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( matrix_file, Comm, readMap,
readA, readx, readb, readxexact) );
} else {
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra( matrix_file, Comm, readMap, readA, readx,
readb, readxexact) ;
}
}
}
Epetra_CrsMatrix transposeA(Copy, *readMap, 0);
Epetra_CrsMatrix *serialA ;
if ( transpose ) {
assert( CrsMatrixTranspose( readA, &transposeA ) == 0 );
serialA = &transposeA ;
} else {
serialA = readA ;
}
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
Epetra_Map* map_;
if( NonContiguousMap ) {
//
// map gives us NumMyElements and MyFirstElement;
//
int NumGlobalElements = readMap->NumGlobalElements();
int NumMyElements = map.NumMyElements();
int MyFirstElement = map.MinMyGID();
std::vector<int> MapMap_( NumGlobalElements );
readMap->MyGlobalElements( &MapMap_[0] ) ;
Comm.Broadcast( &MapMap_[0], NumGlobalElements, 0 ) ;
map_ = new Epetra_Map( NumGlobalElements, NumMyElements, &MapMap_[MyFirstElement], 0, Comm);
} else {
map_ = new Epetra_Map( map ) ;
}
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter(*readMap, *map_);
Epetra_CrsMatrix A(Copy, *map_, 0);
Epetra_RowMatrix * passA = 0;
Epetra_MultiVector * passx = 0;
Epetra_MultiVector * passb = 0;
Epetra_MultiVector * passxexact = 0;
Epetra_MultiVector * passresid = 0;
Epetra_MultiVector * passtmp = 0;
Epetra_MultiVector x(*map_,numsolves);
Epetra_MultiVector b(*map_,numsolves);
Epetra_MultiVector xexact(*map_,numsolves);
Epetra_MultiVector resid(*map_,numsolves);
Epetra_MultiVector tmp(*map_,numsolves);
Epetra_MultiVector serialx(*readMap,numsolves);
Epetra_MultiVector serialb(*readMap,numsolves);
Epetra_MultiVector serialxexact(*readMap,numsolves);
Epetra_MultiVector serialresid(*readMap,numsolves);
Epetra_MultiVector serialtmp(*readMap,numsolves);
bool distribute_matrix = ( matrix_type == AMESOS_Distributed ) ;
if ( distribute_matrix ) {
//
// Initialize x, b and xexact to the values read in from the file
//
A.Export(*serialA, exporter, Add);
Comm.Barrier();
assert(A.FillComplete()==0);
Comm.Barrier();
passA = &A;
passx = &x;
passb = &b;
passxexact = &xexact;
passresid = &resid;
passtmp = &tmp;
} else {
passA = serialA;
passx = &serialx;
passb = &serialb;
passxexact = &serialxexact;
passresid = &serialresid;
passtmp = &serialtmp;
}
passxexact->SetSeed(131) ;
passxexact->Random();
passx->SetSeed(11231) ;
passx->Random();
passb->PutScalar( 0.0 );
passA->Multiply( transpose, *passxexact, *passb ) ;
Epetra_MultiVector CopyB( *passb ) ;
double Anorm = passA->NormInf() ;
SparseDirectTimingVars::SS_Result.Set_Anorm(Anorm) ;
Epetra_LinearProblem Problem( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb );
double max_resid = 0.0;
for ( int j = 0 ; j < special+1 ; j++ ) {
Epetra_Time TotalTime( Comm ) ;
if ( false ) {
#ifdef TEST_UMFPACK
unused code
} else if ( SparseSolver == UMFPACK ) {
UmfpackOO umfpack( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
umfpack.SetTrans( transpose ) ;
umfpack.Solve() ;
#endif
#ifdef TEST_SUPERLU
} else if ( SparseSolver == SuperLU ) {
SuperluserialOO superluserial( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
superluserial.SetPermc( SuperLU_permc ) ;
superluserial.SetTrans( transpose ) ;
superluserial.SetUseDGSSV( special == 0 ) ;
superluserial.Solve() ;
#endif
#ifdef HAVE_AMESOS_SLUD
} else if ( SparseSolver == SuperLUdist ) {
SuperludistOO superludist( Problem ) ;
superludist.SetTrans( transpose ) ;
EPETRA_CHK_ERR( superludist.Solve( true ) ) ;
#endif
#ifdef HAVE_AMESOS_SLUD2
} else if ( SparseSolver == SuperLUdist2 ) {
Superludist2_OO superludist2( Problem ) ;
superludist2.SetTrans( transpose ) ;
EPETRA_CHK_ERR( superludist2.Solve( true ) ) ;
#endif
#ifdef TEST_SPOOLES
} else if ( SparseSolver == SPOOLES ) {
SpoolesOO spooles( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
spooles.SetTrans( transpose ) ;
spooles.Solve() ;
#endif
#ifdef HAVE_AMESOS_DSCPACK
} else if ( SparseSolver == DSCPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Dscpack dscpack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( dscpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( dscpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_UMFPACK
} else if ( SparseSolver == UMFPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Umfpack umfpack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( umfpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( umfpack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( umfpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_KLU
} else if ( SparseSolver == KLU ) {
Teuchos::ParameterList ParamList ;
Amesos_Klu klu( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( klu.SetParameters( ParamList ) );
EPETRA_CHK_ERR( klu.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( klu.SymbolicFactorization( ) );
EPETRA_CHK_ERR( klu.NumericFactorization( ) );
EPETRA_CHK_ERR( klu.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARAKLETE
} else if ( SparseSolver == PARAKLETE ) {
Teuchos::ParameterList ParamList ;
Amesos_Paraklete paraklete( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( paraklete.SetParameters( ParamList ) );
EPETRA_CHK_ERR( paraklete.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( paraklete.SymbolicFactorization( ) );
EPETRA_CHK_ERR( paraklete.NumericFactorization( ) );
EPETRA_CHK_ERR( paraklete.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SLUS
} else if ( SparseSolver == SuperLU ) {
Epetra_SLU superluserial( &Problem ) ;
EPETRA_CHK_ERR( superluserial.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( superluserial.SymbolicFactorization( ) );
EPETRA_CHK_ERR( superluserial.NumericFactorization( ) );
EPETRA_CHK_ERR( superluserial.Solve( ) );
#endif
#ifdef HAVE_AMESOS_LAPACK
} else if ( SparseSolver == LAPACK ) {
Teuchos::ParameterList ParamList ;
ParamList.set( "MaxProcs", -3 );
Amesos_Lapack lapack( Problem ) ;
EPETRA_CHK_ERR( lapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( lapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( lapack.NumericFactorization( ) );
EPETRA_CHK_ERR( lapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_TAUCS
} else if ( SparseSolver == TAUCS ) {
Teuchos::ParameterList ParamList ;
Amesos_Taucs taucs( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( taucs.SetParameters( ParamList ) );
EPETRA_CHK_ERR( taucs.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( taucs.SymbolicFactorization( ) );
EPETRA_CHK_ERR( taucs.NumericFactorization( ) );
EPETRA_CHK_ERR( taucs.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARDISO
} else if ( SparseSolver == PARDISO ) {
Teuchos::ParameterList ParamList ;
Amesos_Pardiso pardiso( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( pardiso.SetParameters( ParamList ) );
EPETRA_CHK_ERR( pardiso.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( pardiso.SymbolicFactorization( ) );
EPETRA_CHK_ERR( pardiso.NumericFactorization( ) );
EPETRA_CHK_ERR( pardiso.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARKLETE
} else if ( SparseSolver == PARKLETE ) {
Teuchos::ParameterList ParamList ;
Amesos_Parklete parklete( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( parklete.SetParameters( ParamList ) );
EPETRA_CHK_ERR( parklete.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( parklete.SymbolicFactorization( ) );
EPETRA_CHK_ERR( parklete.NumericFactorization( ) );
EPETRA_CHK_ERR( parklete.Solve( ) );
#endif
#ifdef HAVE_AMESOS_MUMPS
} else if ( SparseSolver == MUMPS ) {
Teuchos::ParameterList ParamList ;
Amesos_Mumps mumps( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( mumps.SetParameters( ParamList ) );
EPETRA_CHK_ERR( mumps.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( mumps.SymbolicFactorization( ) );
EPETRA_CHK_ERR( mumps.NumericFactorization( ) );
EPETRA_CHK_ERR( mumps.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SCALAPACK
} else if ( SparseSolver == SCALAPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Scalapack scalapack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( scalapack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( scalapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( scalapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( scalapack.NumericFactorization( ) );
EPETRA_CHK_ERR( scalapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SUPERLUDIST
} else if ( SparseSolver == SUPERLUDIST ) {
Teuchos::ParameterList ParamList ;
Amesos_Superludist superludist( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( superludist.SetParameters( ParamList ) );
EPETRA_CHK_ERR( superludist.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( superludist.SymbolicFactorization( ) );
EPETRA_CHK_ERR( superludist.NumericFactorization( ) );
EPETRA_CHK_ERR( superludist.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SUPERLU
} else if ( SparseSolver == SUPERLU ) {
Teuchos::ParameterList ParamList ;
Amesos_Superlu superlu( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( superlu.SetParameters( ParamList ) );
EPETRA_CHK_ERR( superlu.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( superlu.SymbolicFactorization( ) );
EPETRA_CHK_ERR( superlu.NumericFactorization( ) );
EPETRA_CHK_ERR( superlu.Solve( ) );
#endif
#ifdef TEST_SPOOLESSERIAL
} else if ( SparseSolver == SPOOLESSERIAL ) {
SpoolesserialOO spoolesserial( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
spoolesserial.Solve() ;
#endif
} else {
SparseDirectTimingVars::log_file << "Solver not implemented yet" << std::endl ;
std::cerr << "\n\n#################### Requested solver not available (Or not tested with blocked RHS) on this platform #####################\n" << std::endl ;
}
SparseDirectTimingVars::SS_Result.Set_Total_Time( TotalTime.ElapsedTime() );
// SparseDirectTimingVars::SS_Result.Set_First_Time( 0.0 );
// SparseDirectTimingVars::SS_Result.Set_Middle_Time( 0.0 );
// SparseDirectTimingVars::SS_Result.Set_Last_Time( 0.0 );
//
// Compute the error = norm(xcomp - xexact )
//
std::vector <double> error(numsolves) ;
double max_error = 0.0;
passresid->Update(1.0, *passx, -1.0, *passxexact, 0.0);
passresid->Norm2(&error[0]);
for ( int i = 0 ; i< numsolves; i++ )
if ( error[i] > max_error ) max_error = error[i] ;
SparseDirectTimingVars::SS_Result.Set_Error(max_error) ;
// passxexact->Norm2(&error[0] ) ;
// passx->Norm2(&error ) ;
//
// Compute the residual = norm(Ax - b)
//
std::vector <double> residual(numsolves) ;
passtmp->PutScalar(0.0);
passA->Multiply( transpose, *passx, *passtmp);
passresid->Update(1.0, *passtmp, -1.0, *passb, 0.0);
// passresid->Update(1.0, *passtmp, -1.0, CopyB, 0.0);
passresid->Norm2(&residual[0]);
for ( int i = 0 ; i< numsolves; i++ )
if ( residual[i] > max_resid ) max_resid = residual[i] ;
SparseDirectTimingVars::SS_Result.Set_Residual(max_resid) ;
std::vector <double> bnorm(numsolves);
passb->Norm2( &bnorm[0] ) ;
SparseDirectTimingVars::SS_Result.Set_Bnorm(bnorm[0]) ;
std::vector <double> xnorm(numsolves);
passx->Norm2( &xnorm[0] ) ;
SparseDirectTimingVars::SS_Result.Set_Xnorm(xnorm[0]) ;
if ( false && iam == 0 ) {
std::cout << " Amesos_TestMutliSolver.cpp " << std::endl ;
for ( int i = 0 ; i< numsolves && i < 10 ; i++ ) {
std::cout << "i=" << i
<< " error = " << error[i]
<< " xnorm = " << xnorm[i]
<< " residual = " << residual[i]
<< " bnorm = " << bnorm[i]
<< std::endl ;
}
std::cout << std::endl << " max_resid = " << max_resid ;
std::cout << " max_error = " << max_error << std::endl ;
std::cout << " Get_residual() again = " << SparseDirectTimingVars::SS_Result.Get_Residual() << std::endl ;
}
}
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
delete map_;
Comm.Barrier();
return 0 ;
}
// ============================================================================
void EpetraExt::XMLWriter::
Write(const std::string& Label, const Epetra_Map& Map)
{
TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error,
"No file has been opened");
int NumGlobalElements = Map.NumGlobalElements();
int* MyGlobalElements = Map.MyGlobalElements();
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<Map Label=\"" << Label
<< "\" NumElements=\"" << NumGlobalElements << '"'
<< " IndexBase=\"" << Map.IndexBase() << '"'
<< " NumProc=\"" << Comm_.NumProc() << '"';
of.close();
}
for (int iproc = 0; iproc < Comm_.NumProc(); ++iproc)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << " ElementsOnProc" << iproc << "=\"" << Map.NumMyElements() << '"';
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << '>' << std::endl;
of.close();
}
for (int iproc = 0; iproc < Comm_.NumProc(); iproc++)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<Proc ID=\"" << Comm_.MyPID() << "\">" << std::endl;
for (int i = 0; i < Map.NumMyElements(); ++i)
{
of << MyGlobalElements[i] << std::endl;
}
of << "</Proc>" << std::endl;
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "</Map>" << std::endl;
of.close();
}
}
int
main (int argc, char *argv[])
{
using namespace Anasazi;
using Teuchos::RCP;
using Teuchos::rcp;
using std::endl;
#ifdef HAVE_MPI
// Initialize MPI
MPI_Init (&argc, &argv);
#endif // HAVE_MPI
// Create an Epetra communicator
#ifdef HAVE_MPI
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif // HAVE_MPI
// Create an Anasazi output manager
BasicOutputManager<double> printer;
printer.stream(Errors) << Anasazi_Version() << std::endl << std::endl;
// Get the sorting std::string from the command line
std::string which ("LM");
Teuchos::CommandLineProcessor cmdp (false, true);
cmdp.setOption("sort", &which, "Targetted eigenvalues (SM or LM).");
if (cmdp.parse (argc, argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return -1;
}
// Dimension of the matrix
//
// Discretization points in any one direction.
const int nx = 10;
// Size of matrix nx*nx
const int NumGlobalElements = nx*nx;
// Construct a Map that puts approximately the same number of
// equations on each process.
Epetra_Map Map (NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements ();
std::vector<int> MyGlobalElements (NumMyElements);
Map.MyGlobalElements (&MyGlobalElements[0]);
// Create an integer vector NumNz that is used to build the Petra
// matrix. NumNz[i] is the number of OFF-DIAGONAL terms for the
// i-th global equation on this process.
std::vector<int> NumNz (NumMyElements);
/* We are building a matrix of block structure:
| T -I |
|-I T -I |
| -I T |
| ... -I|
| -I T|
where each block is dimension nx by nx and the matrix is on the order of
nx*nx. The block T is a tridiagonal matrix.
*/
for (int i=0; i<NumMyElements; ++i) {
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1 ||
MyGlobalElements[i] == nx-1 || MyGlobalElements[i] == nx*(nx-1) ) {
NumNz[i] = 3;
}
else if (MyGlobalElements[i] < nx || MyGlobalElements[i] > nx*(nx-1) ||
MyGlobalElements[i]%nx == 0 || (MyGlobalElements[i]+1)%nx == 0) {
NumNz[i] = 4;
}
else {
NumNz[i] = 5;
}
}
// Create an Epetra_Matrix
RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, &NumNz[0]));
// Compute coefficients for discrete convection-diffution operator
const double one = 1.0;
std::vector<double> Values(4);
std::vector<int> Indices(4);
double rho = 0.0;
double h = one /(nx+1);
double h2 = h*h;
double c = 5.0e-01*rho/ h;
Values[0] = -one/h2 - c; Values[1] = -one/h2 + c; Values[2] = -one/h2; Values[3]= -one/h2;
double diag = 4.0 / h2;
int NumEntries;
for (int i=0; i<NumMyElements; ++i) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
Indices[1] = nx;
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == nx*(nx-1)) {
Indices[0] = nx*(nx-1)+1;
Indices[1] = nx*(nx-2);
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == nx-1) {
Indices[0] = nx-2;
NumEntries = 1;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = 2*nx-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == NumGlobalElements-1) {
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = nx*(nx-1)-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] < nx) {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] > nx*(nx-1)) {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i]%nx == 0) {
Indices[0] = MyGlobalElements[i]+1;
Indices[1] = MyGlobalElements[i]-nx;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if ((MyGlobalElements[i]+1)%nx == 0) {
Indices[0] = MyGlobalElements[i]-nx;
Indices[1] = MyGlobalElements[i]+nx;
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = MyGlobalElements[i]-1;
NumEntries = 1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
Indices[3] = MyGlobalElements[i]+nx;
NumEntries = 4;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
// Put in the diagonal entry
int info = A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, &MyGlobalElements[i]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
// Finish up
int info = A->FillComplete ();
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "A->FillComplete() returned info = "
<< info << " != 0." );
A->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks
// Create a identity matrix for the temporary mass matrix
RCP<Epetra_CrsMatrix> M = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, 1));
for (int i=0; i<NumMyElements; i++) {
Values[0] = one;
Indices[0] = i;
NumEntries = 1;
info = M->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "M->InsertGlobalValues() returned info = "
<< info << " != 0." );
}
// Finish up
info = M->FillComplete ();
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "M->FillComplete() returned info = "
<< info << " != 0." );
M->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks
//************************************
// Call the LOBPCG solver manager
//***********************************
//
// Variables used for the LOBPCG Method
const int nev = 10;
const int blockSize = 5;
const int maxIters = 500;
const double tol = 1.0e-8;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef MultiVecTraits<double, Epetra_MultiVector> MVT;
// Create an Epetra_MultiVector for an initial vector to start the
// solver. Note: This needs to have the same number of columns as
// the blocksize.
RCP<Epetra_MultiVector> ivec = rcp (new Epetra_MultiVector (Map, blockSize));
ivec->Random (); // fill the initial vector with random values
// Create the eigenproblem.
RCP<BasicEigenproblem<double, MV, OP> > MyProblem =
rcp (new BasicEigenproblem<double, MV, OP> (A, ivec));
// Inform the eigenproblem that the operator A is symmetric
MyProblem->setHermitian (true);
// Set the number of eigenvalues requested
MyProblem->setNEV (nev);
// Tell the eigenproblem that you are finishing passing it information.
const bool success = MyProblem->setProblem ();
if (! success) {
printer.print (Errors, "Anasazi::BasicEigenproblem::setProblem() reported an error.\n");
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return -1;
}
// Create parameter list to pass into the solver manager
Teuchos::ParameterList MyPL;
MyPL.set ("Which", which);
MyPL.set ("Block Size", blockSize);
MyPL.set ("Maximum Iterations", maxIters);
MyPL.set ("Convergence Tolerance", tol);
MyPL.set ("Full Ortho", true);
MyPL.set ("Use Locking", true);
// Create the solver manager
LOBPCGSolMgr<double, MV, OP> MySolverMan (MyProblem, MyPL);
// Solve the problem
ReturnType returnCode = MySolverMan.solve ();
// Get the eigenvalues and eigenvectors from the eigenproblem
Eigensolution<double,MV> sol = MyProblem->getSolution ();
std::vector<Value<double> > evals = sol.Evals;
RCP<MV> evecs = sol.Evecs;
// Compute residuals.
std::vector<double> normR (sol.numVecs);
if (sol.numVecs > 0) {
Teuchos::SerialDenseMatrix<int,double> T (sol.numVecs, sol.numVecs);
Epetra_MultiVector tempAevec (Map, sol.numVecs );
T.putScalar (0.0);
for (int i = 0; i < sol.numVecs; ++i) {
T(i,i) = evals[i].realpart;
}
A->Apply (*evecs, tempAevec);
MVT::MvTimesMatAddMv (-1.0, *evecs, T, 1.0, tempAevec);
MVT::MvNorm (tempAevec, normR);
}
// Print the results
std::ostringstream os;
os.setf (std::ios_base::right, std::ios_base::adjustfield);
os << "Solver manager returned "
<< (returnCode == Converged ? "converged." : "unconverged.") << endl;
os << endl;
os << "------------------------------------------------------" << endl;
os << std::setw(16) << "Eigenvalue"
<< std::setw(18) << "Direct Residual"
<< endl;
os << "------------------------------------------------------" << endl;
for (int i = 0; i < sol.numVecs; ++i) {
os << std::setw(16) << evals[i].realpart
<< std::setw(18) << normR[i] / evals[i].realpart
<< endl;
}
os << "------------------------------------------------------" << endl;
printer.print (Errors, os.str ());
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return 0;
}
int main(int argc, char *argv[]) {
int ierr=0, returnierr=0;
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
if (!verbose) {
Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
}
int MyPID = Comm.MyPID();
int NumProc = Comm.NumProc();
if (verbose && MyPID==0)
cout << Epetra_Version() << endl << endl;
if (verbose) cout << Comm << endl;
bool verbose1 = verbose;
if (verbose) verbose = (MyPID==0);
int NumMyElements = 10000;
int NumMyElements1 = NumMyElements; // Used for local map
int NumGlobalElements = NumMyElements*NumProc+EPETRA_MIN(NumProc,3);
if (MyPID < 3) NumMyElements++;
int IndexBase = 0;
bool DistributedGlobal = (NumGlobalElements>NumMyElements);
Epetra_Map* Map;
// Test exceptions
if (verbose)
cout << "*******************************************************************************************" << endl
<< " Testing Exceptions (Expect error messages if EPETRA_NO_ERROR_REPORTS is not defined" << endl
<< "*******************************************************************************************" << endl
<< endl << endl;
try {
if (verbose) cout << "Checking Epetra_Map(-2, IndexBase, Comm)" << endl;
Map = new Epetra_Map(-2, IndexBase, Comm);
}
catch (int Error) {
if (Error!=-1) {
if (Error!=0) {
EPETRA_TEST_ERR(Error,returnierr);
if (verbose) cout << "Error code should be -1" << endl;
}
else {
cout << "Error code = " << Error << "Should be -1" << endl;
returnierr+=1;
}
}
else if (verbose) cout << "Checked OK\n\n" << endl;
}
try {
if (verbose) cout << "Checking Epetra_Map(2, 3, IndexBase, Comm)" << endl;
Map = new Epetra_Map(2, 3, IndexBase, Comm);
}
catch (int Error) {
if (Error!=-4) {
if (Error!=0) {
EPETRA_TEST_ERR(Error,returnierr);
if (verbose) cout << "Error code should be -4" << endl;
}
else {
cout << "Error code = " << Error << "Should be -4" << endl;
returnierr+=1;
}
}
else if (verbose) cout << "Checked OK\n\n" << endl;
}
if (verbose) cerr << flush;
if (verbose) cout << flush;
Comm.Barrier();
if (verbose)
cout << endl << endl
<< "*******************************************************************************************" << endl
<< " Testing valid constructor now......................................................" << endl
<< "*******************************************************************************************" << endl
<< endl << endl;
// Test Epetra-defined uniform linear distribution constructor
Map = new Epetra_Map(NumGlobalElements, IndexBase, Comm);
if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, IndexBase, Comm)" << endl;
ierr = checkmap(*Map, NumGlobalElements, NumMyElements, 0,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
delete Map;
// Test User-defined linear distribution constructor
Map = new Epetra_Map(NumGlobalElements, NumMyElements, IndexBase, Comm);
if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, NumMyElements, IndexBase, Comm)" << endl;
ierr = checkmap(*Map, NumGlobalElements, NumMyElements, 0,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
delete Map;
// Test User-defined arbitrary distribution constructor
// Generate Global Element List. Do in reverse for fun!
int * MyGlobalElements = new int[NumMyElements];
int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase;
if (Comm.MyPID()>2) MaxMyGID+=3;
for (int i = 0; i<NumMyElements; i++) MyGlobalElements[i] = MaxMyGID-i;
Map = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements,
IndexBase, Comm);
if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm)" << endl;
ierr = checkmap(*Map, NumGlobalElements, NumMyElements, MyGlobalElements,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test Copy constructor
Epetra_Map* Map1 = new Epetra_Map(*Map);
// Test SameAs() method
bool same = Map1->SameAs(*Map);
EPETRA_TEST_ERR(!(same==true),ierr);// should return true since Map1 is a copy of Map
Epetra_BlockMap* Map2 = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm);
same = Map2->SameAs(*Map);
EPETRA_TEST_ERR(!(same==true),ierr); // Map and Map2 were created with the same sets of parameters
delete Map2;
// now test SameAs() on a map that is different
Map2 = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase-1, Comm);
same = Map2->SameAs(*Map);
EPETRA_TEST_ERR(!(same==false),ierr); // IndexBases are different
delete Map2;
// Back to testing copy constructor
if (verbose) cout << "Checking Epetra_Map(*Map)" << endl;
ierr = checkmap(*Map1, NumGlobalElements, NumMyElements, MyGlobalElements,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
Epetra_Map* SmallMap = 0;
if (verbose1) {
// Build a small map for test cout. Use 10 elements from current map
int* MyEls = Map->MyGlobalElements();
int IndBase = Map->IndexBase();
int MyLen = EPETRA_MIN(10+Comm.MyPID(),Map->NumMyElements());
SmallMap = new Epetra_Map(-1, MyLen, MyEls, IndBase, Comm);
}
delete [] MyGlobalElements;
delete Map;
delete Map1;
// Test reference-counting in Epetra_Map
if (verbose) cout << "Checking Epetra_Map reference counting" << endl;
ierr = checkMapDataClass(Comm, verbose);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test LocalMap constructor
Epetra_LocalMap* LocalMap = new Epetra_LocalMap(NumMyElements1, IndexBase, Comm);
if (verbose) cout << "Checking Epetra_LocalMap(NumMyElements1, IndexBase, Comm)" << endl;
ierr = checkmap(*LocalMap, NumMyElements1, NumMyElements1, 0, IndexBase, Comm, false);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test Copy constructor
Epetra_LocalMap* LocalMap1 = new Epetra_LocalMap(*LocalMap);
if (verbose) cout << "Checking Epetra_LocalMap(*LocalMap)" << endl;
ierr = checkmap(*LocalMap1, NumMyElements1, NumMyElements1, 0, IndexBase, Comm, false);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
delete LocalMap1;
delete LocalMap;
// Test reference-counting in Epetra_LocalMap
if (verbose) cout << "Checking Epetra_LocalMap reference counting" << endl;
ierr = checkLocalMapDataClass(Comm, verbose);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test output
if (verbose1) {
if (verbose) cout << "Test ostream << operator" << endl << flush;
cout << *SmallMap;
delete SmallMap;
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return returnierr;
}
