bool probing_test(Epetra_CrsMatrix & in_mat, bool build_list){
const Epetra_CrsGraph & graph=in_mat.Graph();
Teuchos::ParameterList main, zoltan;
if(build_list){
// zoltan.set("DISTANCE","2");
main.set("ZOLTAN",zoltan);
}
Isorropia::Epetra::Prober prober(Teuchos::rcp<const Epetra_CrsGraph>(&graph,false),main);
Epetra_CrsMatrix out_mat(Copy,graph);
int rv=prober.probe(in_mat,out_mat);
if(rv!=0) {printf("ERROR: probing failed\n");return false;}
#ifdef HAVE_EPETRAEXT
EpetraExt::MatrixMatrix::Add(in_mat,false,1,out_mat,-1);
double nrm=out_mat.NormInf()/in_mat.NormInf();
if(!in_mat.Comm().MyPID())
printf("diff norm = %22.16e\n",nrm);
if(nrm < 1e-12) return true;
else return false;
#endif
return true;
}
// ================================================ ====== ==== ==== == =
// Computes C= <me> * A
int ML_Epetra::ML_RefMaxwell_11_Operator::MatrixMatrix_Multiply(const Epetra_CrsMatrix & A, Epetra_CrsMatrix **C) const
{
ML_Comm* comm;
ML_Comm_Create(&comm);
#ifdef ML_MPI
const Epetra_MpiComm *epcomm = dynamic_cast<const Epetra_MpiComm*>(&(A.Comm()));
// Get the MPI communicator, as it may not be MPI_COMM_W0RLD, and update the ML comm object
if (epcomm) ML_Comm_Set_UsrComm(comm,epcomm->Comm());
#endif
ML_Operator *C_;
int rv=MatrixMatrix_Multiply(A,comm,&C_);
Epetra_CrsMatrix_Wrap_ML_Operator(C_,*Comm_,*DomainMap_,C,Copy,A.IndexBase());
ML_Operator_Destroy(&C_);
ML_Comm_Destroy(&comm);
return rv;
}/*end MatrixMatrix_Multiply*/
//==============================================================================
Ifpack_CrsIct::Ifpack_CrsIct(const Epetra_CrsMatrix & A, double Droptol, int Lfil)
: A_(A),
Comm_(A.Comm()),
Allocated_(false),
ValuesInitialized_(false),
Factored_(false),
Condest_(-1.0),
Athresh_(0.0),
Rthresh_(1.0),
Droptol_(Droptol),
Lfil_(Lfil),
LevelOverlap_(0),
OverlapMode_(Zero),
Aict_(0),
Lict_(0),
Ldiag_(0)
{
Allocate();
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_CrsMatrix* A = dynamic_cast<Epetra_CrsMatrix*>(Problem.GetMatrix());
int PID = A->Comm().MyPID();
int numProcs = A->Comm().NumProc();
RCP<const Epetra_RowMatrix> Arcp = Teuchos::rcp(A, false);
double n1, n2,nf;
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
MLList.set("ML output", 0);
RowMatrixToMatlabFile("mat_f.dat",*A);
MultiVectorToMatrixMarketFile("lhs_f.dat",*lhs,0,0,false);
MultiVectorToMatrixMarketFile("rhs_f.dat",*rhs,0,0,false);
Epetra_Time Time(A->Comm());
/* Build the Zoltan list - Group #1 */
ParameterList Zlist1,Sublist1;
Sublist1.set("DEBUG_LEVEL","0");
Sublist1.set("NUM_GLOBAL_PARTITIONS","2");
Zlist1.set("Zoltan",Sublist1);
/* Start Isorropia's Ninja Magic - Group #1 */
RefCountPtr<Isorropia::Epetra::Partitioner> partitioner1 =
Isorropia::Epetra::create_partitioner(Arcp, Zlist1);
Isorropia::Epetra::Redistributor rd1(partitioner1);
Teuchos::RCP<Epetra_CrsMatrix> ResA1=rd1.redistribute(*A);
Teuchos::RCP<Epetra_MultiVector> ResX1=rd1.redistribute(*lhs);
Teuchos::RCP<Epetra_MultiVector> ResB1=rd1.redistribute(*rhs);
RestrictedCrsMatrixWrapper RW1;
RW1.restrict_comm(ResA1);
RestrictedMultiVectorWrapper RX1,RB1;
RX1.restrict_comm(ResX1);
RB1.restrict_comm(ResB1);
/* Build the Zoltan list - Group #2 */
ParameterList Zlist2,Sublist2;
Sublist2.set("DEBUG_LEVEL","0");
if(PID > 1) Sublist2.set("NUM_LOCAL_PARTITIONS","1");
else Sublist2.set("NUM_LOCAL_PARTITIONS","0");
Zlist2.set("Zoltan",Sublist2);
/* Start Isorropia's Ninja Magic - Group #2 */
RefCountPtr<Isorropia::Epetra::Partitioner> partitioner2 =
Isorropia::Epetra::create_partitioner(Arcp, Zlist2);
Isorropia::Epetra::Redistributor rd2(partitioner2);
Teuchos::RCP<Epetra_CrsMatrix> ResA2=rd2.redistribute(*A);
Teuchos::RCP<Epetra_MultiVector> ResX2=rd2.redistribute(*lhs);
Teuchos::RCP<Epetra_MultiVector> ResB2=rd2.redistribute(*rhs);
RestrictedCrsMatrixWrapper RW2;
RW2.restrict_comm(ResA2);
RestrictedMultiVectorWrapper RX2,RB2;
RX2.restrict_comm(ResX2);
RB2.restrict_comm(ResB2);
if(RW1.RestrictedProcIsActive()){
Teuchos::RCP<Epetra_CrsMatrix> SubA1 = RW1.RestrictedMatrix();
Teuchos::RCP<Epetra_MultiVector> SubX1 = RX1.RestrictedMultiVector();
Teuchos::RCP<Epetra_MultiVector> SubB1 = RB1.RestrictedMultiVector();
ML_Epetra::MultiLevelPreconditioner * SubPrec1 = new ML_Epetra::MultiLevelPreconditioner(*SubA1, MLList, true);
Epetra_LinearProblem Problem1(&*SubA1,&*SubX1,&*SubB1);
AztecOO solver1(Problem1);
solver1.SetPrecOperator(SubPrec1);
solver1.SetAztecOption(AZ_solver, AZ_gmres);
solver1.SetAztecOption(AZ_output, 32);
solver1.SetAztecOption(AZ_kspace, 160);
solver1.Iterate(1550, 1e-12);
delete SubPrec1;
}
else{
Teuchos::RCP<Epetra_CrsMatrix> SubA2 = RW2.RestrictedMatrix();
Teuchos::RCP<Epetra_MultiVector> SubX2 = RX2.RestrictedMultiVector();
Teuchos::RCP<Epetra_MultiVector> SubB2 = RB2.RestrictedMultiVector();
ML_Epetra::MultiLevelPreconditioner * SubPrec2 = new ML_Epetra::MultiLevelPreconditioner(*SubA2, MLList, true);
Epetra_LinearProblem Problem2(&*SubA2,&*SubX2,&*SubB2);
AztecOO solver2(Problem2);
solver2.SetPrecOperator(SubPrec2);
solver2.SetAztecOption(AZ_solver, AZ_gmres);
solver2.SetAztecOption(AZ_output, 32);
solver2.SetAztecOption(AZ_kspace, 160);
solver2.Iterate(1550, 1e-12);
delete SubPrec2;
}
/* Post-processing exports */
Epetra_MultiVector ans1(*lhs), ans2(*lhs);
rd1.redistribute_reverse(*ResX1,ans1);
rd2.redistribute_reverse(*ResX2,ans2);
/* Run on Full Problem */
A->Comm().Barrier();
ML_Epetra::MultiLevelPreconditioner * FullPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
AztecOO solverF(Problem);
solverF.SetPrecOperator(FullPrec);
solverF.SetAztecOption(AZ_solver, AZ_gmres);
solverF.SetAztecOption(AZ_output, 32);
solverF.SetAztecOption(AZ_kspace, 160);
solverF.Iterate(1550, 1e-12);
delete FullPrec;
/* Solution Comparison */
ans1.Update(1.0,*lhs,-1.0);
ans2.Update(1.0,*lhs,-1.0);
ans1.Norm2(&n1);
ans2.Norm2(&n2);
if(!PID) {
printf("Norm Diff 1 = %6.4e\n",n1);
printf("Norm Diff 2 = %6.4e\n",n2);
}
TotalErrorExactSol += n1 + n2;
}
bool CrsMatrixInfo( const Epetra_CrsMatrix & A,
ostream & os )
{
int MyPID = A.Comm().MyPID();
// take care that matrix is already trasformed
bool IndicesAreGlobal = A.IndicesAreGlobal();
if( IndicesAreGlobal == true ) {
if( MyPID == 0 ) {
os << "WARNING : matrix must be transformed to local\n";
os << " before calling CrsMatrixInfo\n";
os << " Now returning...\n";
}
return false;
}
int NumGlobalRows = A.NumGlobalRows();
int NumGlobalNonzeros = A.NumGlobalNonzeros();
int NumGlobalCols = A.NumGlobalCols();
double NormInf = A.NormInf();
double NormOne = A.NormOne();
int NumGlobalDiagonals = A.NumGlobalDiagonals();
int GlobalMaxNumEntries = A.GlobalMaxNumEntries();
int IndexBase = A.IndexBase();
bool StorageOptimized = A.StorageOptimized();
bool LowerTriangular = A.LowerTriangular();
bool UpperTriangular = A.UpperTriangular();
bool NoDiagonal = A.NoDiagonal();
// these variables identifies quantities I have to compute,
// since not provided by Epetra_CrsMatrix
double MyFrobeniusNorm( 0.0 ), FrobeniusNorm( 0.0 );
double MyMinElement( DBL_MAX ), MinElement( DBL_MAX );
double MyMaxElement( DBL_MIN ), MaxElement( DBL_MIN );
double MyMinAbsElement( DBL_MAX ), MinAbsElement( DBL_MAX );
double MyMaxAbsElement( 0.0 ), MaxAbsElement( 0.0 );
int NumMyRows = A.NumMyRows();
int * NzPerRow = new int[NumMyRows];
int Row; // iterator on rows
int Col; // iterator on cols
int MaxNumEntries = A.MaxNumEntries();
double * Values = new double[MaxNumEntries];
int * Indices = new int[MaxNumEntries];
double Element, AbsElement; // generic nonzero element and its abs value
int NumEntries;
double * Diagonal = new double [NumMyRows];
// SumOffDiagonal is the sum of absolute values for off-diagonals
double * SumOffDiagonal = new double [NumMyRows];
for( Row=0 ; Row<NumMyRows ; ++Row ) {
SumOffDiagonal[Row] = 0.0;
}
int * IsDiagonallyDominant = new int [NumMyRows];
int GlobalRow;
// cycle over all matrix elements
for( Row=0 ; Row<NumMyRows ; ++Row ) {
GlobalRow = A.GRID(Row);
NzPerRow[Row] = A.NumMyEntries(Row);
A.ExtractMyRowCopy(Row,NzPerRow[Row],NumEntries,Values,Indices);
for( Col=0 ; Col<NumEntries ; ++Col ) {
Element = Values[Col];
AbsElement = abs(Element);
if( Element<MyMinElement ) MyMinElement = Element;
if( Element>MyMaxElement ) MyMaxElement = Element;
if( AbsElement<MyMinAbsElement ) MyMinAbsElement = AbsElement;
if( AbsElement>MyMaxAbsElement ) MyMaxAbsElement = AbsElement;
if( Indices[Col] == Row ) Diagonal[Row] = Element;
else
SumOffDiagonal[Row] += abs(Element);
MyFrobeniusNorm += pow(Element,2);
}
}
// analise storage per row
int MyMinNzPerRow( NumMyRows ), MinNzPerRow( NumMyRows );
int MyMaxNzPerRow( 0 ), MaxNzPerRow( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( NzPerRow[Row]<MyMinNzPerRow ) MyMinNzPerRow=NzPerRow[Row];
if( NzPerRow[Row]>MyMaxNzPerRow ) MyMaxNzPerRow=NzPerRow[Row];
}
// a test to see if matrix is diagonally-dominant
int MyDiagonalDominance( 0 ), DiagonalDominance( 0 );
int MyWeakDiagonalDominance( 0 ), WeakDiagonalDominance( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( abs(Diagonal[Row])>SumOffDiagonal[Row] )
++MyDiagonalDominance;
else if( abs(Diagonal[Row])==SumOffDiagonal[Row] )
++MyWeakDiagonalDominance;
}
// reduction operations
A.Comm().SumAll(&MyFrobeniusNorm, &FrobeniusNorm, 1);
A.Comm().MinAll(&MyMinElement, &MinElement, 1);
A.Comm().MaxAll(&MyMaxElement, &MaxElement, 1);
A.Comm().MinAll(&MyMinAbsElement, &MinAbsElement, 1);
A.Comm().MaxAll(&MyMaxAbsElement, &MaxAbsElement, 1);
A.Comm().MinAll(&MyMinNzPerRow, &MinNzPerRow, 1);
A.Comm().MaxAll(&MyMaxNzPerRow, &MaxNzPerRow, 1);
A.Comm().SumAll(&MyDiagonalDominance, &DiagonalDominance, 1);
A.Comm().SumAll(&MyWeakDiagonalDominance, &WeakDiagonalDominance, 1);
// free memory
delete Values;
delete Indices;
delete Diagonal;
delete SumOffDiagonal;
delete IsDiagonallyDominant;
delete NzPerRow;
// simply no output for MyPID>0, only proc 0 write on os
if( MyPID != 0 ) return true;
os << "*** general Information about the matrix\n";
os << "Number of Global Rows = " << NumGlobalRows << endl;
os << "Number of Global Cols = " << NumGlobalCols << endl;
os << "is the matrix square = " <<
((NumGlobalRows==NumGlobalCols)?"yes":"no") << endl;
os << "||A||_\\infty = " << NormInf << endl;
os << "||A||_1 = " << NormOne << endl;
os << "||A||_F = " << sqrt(FrobeniusNorm) << endl;
os << "Number of nonzero diagonal entries = "
<< NumGlobalDiagonals
<< "( " << 1.0* NumGlobalDiagonals/NumGlobalRows*100
<< " %)\n";
os << "Nonzero per row : min = " << MinNzPerRow
<< " average = " << 1.0*NumGlobalNonzeros/NumGlobalRows
<< " max = " << MaxNzPerRow << endl;
os << "Maximum number of nonzero elements/row = "
<< GlobalMaxNumEntries << endl;
os << "min( a_{i,j} ) = " << MinElement << endl;
os << "max( a_{i,j} ) = " << MaxElement << endl;
os << "min( abs(a_{i,j}) ) = " << MinAbsElement << endl;
os << "max( abs(a_{i,j}) ) = " << MaxAbsElement << endl;
os << "Number of diagonal dominant rows = " << DiagonalDominance
<< " (" << 100.0*DiagonalDominance/NumGlobalRows << " % of total)\n";
os << "Number of weakly diagonal dominant rows = "
<< WeakDiagonalDominance
<< " (" << 100.0*WeakDiagonalDominance/NumGlobalRows << " % of total)\n";
os << "*** Information about the Trilinos storage\n";
os << "Base Index = " << IndexBase << endl;
os << "is storage optimized = "
<< ((StorageOptimized==true)?"yes":"no") << endl;
os << "are indices global = "
<< ((IndicesAreGlobal==true)?"yes":"no") << endl;
os << "is matrix lower triangular = "
<< ((LowerTriangular==true)?"yes":"no") << endl;
os << "is matrix upper triangular = "
<< ((UpperTriangular==true)?"yes":"no") << endl;
os << "are there diagonal entries = "
<< ((NoDiagonal==false)?"yes":"no") << endl;
return true;
}
/* Computes the approximate Schur complement for the wide separator
using guided probing*/
Teuchos::RCP<Epetra_CrsMatrix> computeSchur_GuidedProbing
(
shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure
Epetra_Map *localDRowMap
)
{
int i;
double relative_thres = config->relative_threshold;
Epetra_CrsMatrix *G = ssym->G.getRawPtr();
Epetra_CrsMatrix *R = ssym->R.getRawPtr();
Epetra_LinearProblem *LP = ssym->LP.getRawPtr();
Amesos_BaseSolver *solver = ssym->Solver.getRawPtr();
Ifpack_Preconditioner *ifSolver = ssym->ifSolver.getRawPtr();
Epetra_CrsMatrix *C = ssym->C.getRawPtr();
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
Epetra_Map CrMap = C->RowMap();
int *c_rows = CrMap.MyGlobalElements();
int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
int c_localElems = CrMap.NumMyElements();
int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
Epetra_Map RrMap = R->RowMap();
int *r_rows = RrMap.MyGlobalElements();
int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
int r_localElems = RrMap.NumMyElements();
int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map C_localRMap (-1, c_localElems, c_rows, 0, LComm);
Epetra_Map C_localCMap (-1, c_localcolElems, c_cols, 0, LComm);
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
Epetra_Map R_localRMap (-1, r_localElems, r_rows, 0, LComm);
Epetra_Map R_localCMap (-1, r_localcolElems, r_cols, 0, LComm);
//cout << "#local rows" << g_localElems << "#non zero local cols" << c_localcolElems << endl;
#ifdef DEBUG
cout << "DEBUG MODE" << endl;
int nrows = C->RowMap().NumMyElements();
assert(nrows == localDRowMap->NumGlobalElements());
int gids[nrows], gids1[nrows];
C_localRMap.MyGlobalElements(gids);
localDRowMap->MyGlobalElements(gids1);
cout << "Comparing R's domain map with D's row map" << endl;
for (int i = 0; i < nrows; i++)
{
assert(gids[i] == gids1[i]);
}
#endif
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
//cout << "Creating local matrices" << endl;
int err;
Epetra_CrsMatrix localC(Copy, C_localRMap, C->MaxNumEntries(), false);
for (i = 0; i < c_localElems ; i++)
{
gid = c_rows[i];
err = C->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1,
indices1);
assert (err == 0);
//if (nentries1 > 0) // TODO: Later
//{
err = localC.InsertGlobalValues(gid, nentries1, values1, indices1);
assert (err == 0);
//}
}
localC.FillComplete(G_localRMap, C_localRMap);
//cout << "Created local C matrix" << endl;
Epetra_CrsMatrix localR(Copy, R_localRMap, R->MaxNumEntries(), false);
for (i = 0; i < r_localElems ; i++)
{
gid = r_rows[i];
R->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
localR.InsertGlobalValues(gid, nentries1, values1, indices1);
}
localR.FillComplete(*localDRowMap, R_localRMap);
//cout << "Created local R matrix" << endl;
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
cout << "Created local G matrix" << endl;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, ssym, &localG, &localR, LP, solver,
ifSolver, &localC, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
Teuchos::RCP<Epetra_CrsGraph> lSGraph = Teuchos::RCP<Epetra_CrsGraph> (
new Epetra_CrsGraph(Copy, G_localRMap, maxentries));
if (data->num_compute % config->reset_iter == 0)
{
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[g_localElems];
int *indices = new int[g_localElems];
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec(G_localRMap, nvectors);
Epetra_MultiVector Scol(G_localRMap, nvectors);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
probevec.PutScalar(0.0); // TODO: Move it out
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
// Not much of use for Shasta 2x2 .. Later.
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[cindex] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0)
{
dropped++;
//cout << "vecvalues[j]" << vecvalues[j] <<
// " max" << maxvalue[k] << endl;
}
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < g_localElems ; i++)
{
// TODO: Can move the next two decalarations outside the loop
Epetra_MultiVector probevec(G_localRMap, 1);
Epetra_MultiVector Scol(G_localRMap, 1);
probevec.PutScalar(0.0);
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[i], 0, 1.0);
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[i] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
probeop.PrintTimingInfo();
Sbar->FillComplete();
lSGraph->FillComplete();
data->localSbargraph = lSGraph;
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
cout << "#dropped entries" << dropped << endl;
delete[] values;
delete[] indices;
delete[] maxvalue;
}
else
{
if (((data->num_compute-1) % config->reset_iter) == 0)
{
// We recomputed the Schur complement with dropping for the last
// compute. Reset the prober with the new orthogonal vectors for
// the Sbar from the previous iteration.
Teuchos::ParameterList pList;
Teuchos::RCP<Isorropia::Epetra::Prober> gprober =
Teuchos::RCP<Isorropia::Epetra::Prober> (new
Isorropia::Epetra::Prober(
data->localSbargraph.getRawPtr(), pList, false));
gprober->color();
data->guided_prober = gprober;
}
// Use the prober to probe the probeop for the sparsity pattern
// add that to Sbar and call Fill complete
int nvectors = data->guided_prober->getNumOrthogonalVectors();
cout << "Number of Orthogonal Vectors for guided probing" << nvectors
<< endl;
probeop.ResetTempVectors(nvectors);
Teuchos::RCP<Epetra_CrsMatrix> blockdiag_Sbar =
data->guided_prober->probe(probeop);
int maxentries = blockdiag_Sbar->GlobalMaxNumEntries();
int *indices = new int[maxentries];
double *values = new double[maxentries];
int numentries;
for (int i = 0; i < blockdiag_Sbar->NumGlobalRows() ; i++)
{
int gid = blockdiag_Sbar->GRID(i);
blockdiag_Sbar->ExtractGlobalRowCopy(gid, maxentries, numentries,
values, indices);
Sbar->InsertGlobalValues(gid, numentries, values, indices);
}
Sbar->FillComplete();
delete[] indices;
delete[] values;
}
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
return Sbar;
}
int
NOX::Epetra::TestCompare::testCrsMatrices(
const Epetra_CrsMatrix& mat ,
const Epetra_CrsMatrix& mat_expected ,
double rtol, double atol ,
const std::string& name ,
bool enforceStructure )
{
if (utils.isPrintType(NOX::Utils::TestDetails))
os << std::endl << "\tChecking " << name << ": ";
int passed = 0;
if( !mat_expected.RowMap().SameAs( mat.RowMap() ) )
{
passed = 1;
os << "Failed." << std::endl;
os << std::endl << "\t\tRow maps are not compatible." << std::endl;
return passed;
}
int numEntries1, numEntries2 ;
int * columns1 , * columns2 ;
double * values1 , * values2 ;
int chkSize = 0 ;
double maxVal = 0.0 ;
double infNorm = 0.0 ;
for( int row = 0; row < mat_expected.NumMyRows(); ++row )
{
mat_expected.ExtractMyRowView(row, numEntries1, values1, columns1);
mat.ExtractMyRowView (row, numEntries2, values2, columns2);
if( numEntries1 != numEntries2 )
{
if( enforceStructure )
{
os << std::endl << "\t\t\t";
}
else
os << std::endl << "\t\tWARNING: ";
os << "Matrix size is incompatible for Local Row " << row
<< "\n\t\t\t..... expected " << numEntries1 << " columns, found " << numEntries2
<< std::endl;
chkSize = 1;
}
mat.Comm().SumAll( &chkSize, &passed, 1 );
if( 0 != passed )
{
if( enforceStructure )
{
os << "Failed." << std::endl;
return passed;
}
else
{
chkSize = 0;
passed = 0;
}
}
// Comapre column indices and values
int baseCol = 0 ;
int testCol = 0 ;
int chkCol = 0 ;
double baseVal = 0.0 ;
double testVal = 0.0 ;
double chkVal = 0.0 ;
for( int col = 0; col < numEntries1; ++col )
{
baseCol = columns1[col];
testCol = columns2[col];
baseVal = values1 [col];
testVal = values2 [col];
if( baseCol != testCol )
{
if( enforceStructure )
{
os << std::endl << "\t\t\t";
}
else
os << std::endl << "\t\tWARNING: ";
os << "Column index for Local Row " << row << " is incompatible."
<< "\n\t\t\t..... expected " << baseCol << " , found " << testCol << std::endl;
chkCol = 1;
}
mat.Comm().SumAll( &chkCol, &passed, 1 );
if( 0 != passed )
{
if( enforceStructure )
{
os << "Failed." << std::endl;
return passed;
}
else
{
chkCol = 0;
passed = 0;
continue; // skip pvalue check
}
}
chkVal = fabs( testVal - baseVal ) / (atol + rtol * fabs(baseVal));
mat.Comm().MaxAll( &chkVal, &maxVal, 1 );
if( maxVal > infNorm )
infNorm = maxVal;
if( 1 < maxVal )
break;
}
if( 1 < maxVal )
break;
}
if( 1 < maxVal )
passed = 1; // false by convention in NOX::TestCompare
else
passed = 0; // true by convention in NOX::TestCompare
if (utils.isPrintType(NOX::Utils::TestDetails))
{
if( 0 == passed)
os << "Passed." << std::endl;
else
os << "Failed." << std::endl;
os << "\t\tComputed norm: " << utils.sciformat(infNorm)
<< std::endl
<< "\t\tRelative Tolerance: " << utils.sciformat(rtol)
<< std::endl
<< "\t\tAbsolute Tolerance: " << utils.sciformat(rtol)
<< std::endl;
}
return passed;
}
//
// Diagonal: 0=no change, 1=eliminate entry
// from the map for the largest row element in process 0
// 2=add diagonal entries to the matrix, with a zero value
// (assume row map contains all diagonal entries).
//
// ReindexRowMap:
// 0=no change, 1= add 2 (still contiguous), 2=non-contiguous
//
// ReindexColMap
// 0=same as RowMap, 1=add 4 - Different From RowMap, but contiguous)
//
// RangeMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
// DomainMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
RCP<Epetra_CrsMatrix> NewMatNewMap(Epetra_CrsMatrix& In,
int Diagonal,
int ReindexRowMap,
int ReindexColMap,
int RangeMapType,
int DomainMapType
)
{
//
// If we are making no change, return the original matrix (which has a linear map)
//
#if 0
std::cout << __FILE__ << "::" << __LINE__ << " "
<< Diagonal << " "
<< ReindexRowMap << " "
<< ReindexColMap << " "
<< RangeMapType << " "
<< DomainMapType << " " << std::endl ;
#endif
if ( Diagonal + ReindexRowMap + ReindexColMap + RangeMapType + DomainMapType == 0 ) {
RCP<Epetra_CrsMatrix> ReturnOrig = rcp( &In, false );
return ReturnOrig ;
}
//
// Diagonal==2 is used for a different purpose -
// Making sure that the diagonal of the matrix is non-empty.
// Note: The diagonal must exist in In.RowMap().
//
if ( Diagonal == 2 ) {
assert( ReindexRowMap==0 && ReindexColMap == 0 ) ;
}
int (*RowPermute)(int in) = 0;
int (*ColPermute)(int in) = 0;
assert( Diagonal >= 0 && Diagonal <= 2 );
assert( ReindexRowMap>=0 && ReindexRowMap<=2 );
assert( ReindexColMap>=0 && ReindexColMap<=1 );
assert( RangeMapType>=0 && RangeMapType<=3 );
assert( DomainMapType>=0 && DomainMapType<=3 );
Epetra_Map DomainMap = In.DomainMap();
Epetra_Map RangeMap = In.RangeMap();
Epetra_Map ColMap = In.ColMap();
Epetra_Map RowMap = In.RowMap();
int NumMyRowElements = RowMap.NumMyElements();
int NumMyColElements = ColMap.NumMyElements();
int NumMyRangeElements = RangeMap.NumMyElements();
int NumMyDomainElements = DomainMap.NumMyElements();
int NumGlobalRowElements = RowMap.NumGlobalElements();
int NumGlobalColElements = ColMap.NumGlobalElements();
int NumGlobalRangeElements = RangeMap.NumGlobalElements();
int NumGlobalDomainElements = DomainMap.NumGlobalElements();
assert( NumGlobalRangeElements == NumGlobalDomainElements ) ;
std::vector<int> MyGlobalRowElements( NumMyRowElements ) ;
std::vector<int> NumEntriesPerRow( NumMyRowElements ) ;
std::vector<int> MyPermutedGlobalRowElements( NumMyRowElements ) ;
std::vector<int> MyGlobalColElements( NumMyColElements ) ;
std::vector<int> MyPermutedGlobalColElements( NumMyColElements ) ; // Used to create the column map
std::vector<int> MyPermutedGlobalColElementTable( NumMyColElements ) ; // To convert local indices to global
std::vector<int> MyGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyPermutedGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyGlobalDomainElements( NumMyDomainElements ) ;
std::vector<int> MyPermutedGlobalDomainElements( NumMyDomainElements ) ;
RowMap.MyGlobalElements(&MyGlobalRowElements[0]);
ColMap.MyGlobalElements(&MyGlobalColElements[0]);
RangeMap.MyGlobalElements(&MyGlobalRangeElements[0]);
DomainMap.MyGlobalElements(&MyGlobalDomainElements[0]);
switch( ReindexRowMap ) {
case 0:
RowPermute = &NoPermute ;
break;
case 1:
RowPermute = &SmallRowPermute ;
break;
case 2:
RowPermute = BigRowPermute ;
break;
}
switch( ReindexColMap ) {
case 0:
ColPermute = RowPermute ;
break;
case 1:
ColPermute = &SmallColPermute ;
break;
}
//
// Create Serial Range and Domain Maps based on the permuted indexing
//
int nlocal = 0;
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements;
std::vector<int> AllIDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDs[i] = (*RowPermute)( i ) ;
Epetra_Map SerialRangeMap( -1, nlocal, &AllIDs[0], 0, In.Comm());
std::vector<int> AllIDBs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDBs[i] = (*ColPermute)( i ) ;
Epetra_Map SerialDomainMap( -1, nlocal, &AllIDBs[0], 0, In.Comm());
//
// Create Bizarre Range and Domain Maps based on the permuted indexing
// These are nearly serial, having all but one element on process 0
// The goal here is to make sure that we can use Domain and Range maps
// that are neither serial, nor distributed in the normal manner.
//
std::vector<int> AllIDCs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDCs[i] = (*ColPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDCs[0] = (*ColPermute)( NumGlobalRangeElements - 1 );
}
}
int iam = In.Comm().MyPID();
Epetra_Map BizarreDomainMap( -1, nlocal, &AllIDCs[0], 0, In.Comm());
std::vector<int> AllIDDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDDs[i] = (*RowPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDDs[0] = (*RowPermute)( NumGlobalRangeElements -1 ) ;
}
}
Epetra_Map BizarreRangeMap( -1, nlocal, &AllIDDs[0], 0, In.Comm());
//
// Compute the column map
//
// If Diagonal==1, remove the column corresponding to the last row owned
// by process 0. Removing this column from a tridiagonal matrix, leaves
// a disconnected, but non-singular matrix.
//
int NumMyColElementsOut = 0 ;
int NumGlobalColElementsOut ;
if ( Diagonal == 1 )
NumGlobalColElementsOut = NumGlobalColElements-1;
else
NumGlobalColElementsOut = NumGlobalColElements;
if ( Diagonal == 1 && iam==0 ) {
for ( int i=0; i < NumMyColElements ; i++ ) {
if ( MyGlobalColElements[i] != MyGlobalRowElements[NumMyRowElements-1] ) {
MyPermutedGlobalColElements[NumMyColElementsOut++] =
(*ColPermute)( MyGlobalColElements[i] ) ;
}
}
assert( NumMyColElementsOut == NumMyColElements-1 );
} else {
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElements[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
NumMyColElementsOut = NumMyColElements ;
if ( Diagonal == 2 ) {
// For each row, make sure that the column map has this row in it,
// if it doesn't, add it to the column map.
// Note: MyPermutedGlobalColElements == MyGlobalColElements when
// Diagonal==2 because ( Diagonal == 2 ) implies:
// ReindexRowMap==0 && ReindexColMap == 0 - see assert above
for ( int i=0; i < NumMyRowElements ; i++ ) {
bool MissingDiagonal = true;
for ( int j=0; j < NumMyColElements; j++ ) {
if ( MyGlobalRowElements[i] == MyGlobalColElements[j] ) {
MissingDiagonal = false;
}
}
if ( MissingDiagonal ) {
MyPermutedGlobalColElements.resize(NumMyColElements+1);
MyPermutedGlobalColElements[NumMyColElementsOut] = MyGlobalRowElements[i];
NumMyColElementsOut++;
}
}
In.Comm().SumAll(&NumMyColElementsOut,&NumGlobalColElementsOut,1);
}
}
//
// These tables are used both as the permutation tables and to create the maps.
//
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElementTable[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
for ( int i=0; i < NumMyRowElements ; i++ )
MyPermutedGlobalRowElements[i] =
(*RowPermute)( MyGlobalRowElements[i] ) ;
for ( int i=0; i < NumMyRangeElements ; i++ )
MyPermutedGlobalRangeElements[i] =
(*RowPermute)( MyGlobalRangeElements[i] ) ;
for ( int i=0; i < NumMyDomainElements ; i++ )
MyPermutedGlobalDomainElements[i] =
(*ColPermute)( MyGlobalDomainElements[i] ) ;
RCP<Epetra_Map> PermutedRowMap =
rcp( new Epetra_Map( NumGlobalRowElements, NumMyRowElements,
&MyPermutedGlobalRowElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedColMap =
rcp( new Epetra_Map( NumGlobalColElementsOut, NumMyColElementsOut,
&MyPermutedGlobalColElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedRangeMap =
rcp( new Epetra_Map( NumGlobalRangeElements, NumMyRangeElements,
&MyPermutedGlobalRangeElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedDomainMap =
rcp( new Epetra_Map( NumGlobalDomainElements, NumMyDomainElements,
&MyPermutedGlobalDomainElements[0], 0, In.Comm() ) );
//
// These vectors are filled and then passed to InsertGlobalValues
//
std::vector<int> ThisRowIndices( In.MaxNumEntries() );
std::vector<double> ThisRowValues( In.MaxNumEntries() );
std::vector<int> PermutedGlobalColIndices( In.MaxNumEntries() );
//std::cout << __FILE__ << "::" <<__LINE__ << std::endl ;
RCP<Epetra_CrsMatrix> Out =
rcp( new Epetra_CrsMatrix( Copy, *PermutedRowMap, *PermutedColMap, 0 ) );
for (int i=0; i<NumMyRowElements; i++)
{
int NumIndicesThisRow = 0;
assert( In.ExtractMyRowCopy( i,
In.MaxNumEntries(),
NumIndicesThisRow,
&ThisRowValues[0],
&ThisRowIndices[0] ) == 0 ) ;
for (int j = 0 ; j < NumIndicesThisRow ; j++ )
{
PermutedGlobalColIndices[j] = MyPermutedGlobalColElementTable[ ThisRowIndices[j] ] ;
}
bool MissingDiagonal = false;
if ( Diagonal==2 ) {
//
assert( MyGlobalRowElements[i] == MyPermutedGlobalRowElements[i] );
MissingDiagonal = true;
for( int j =0 ; j < NumIndicesThisRow ; j++ ) {
if ( PermutedGlobalColIndices[j] == MyPermutedGlobalRowElements[i] ) {
MissingDiagonal = false ;
}
}
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< " MyPermutedGlobalRowElements[i] = " << MyPermutedGlobalRowElements[i]
<< " MissingDiagonal = " << MissingDiagonal << std::endl ;
#endif
}
if ( MissingDiagonal ) {
ThisRowValues.resize(NumIndicesThisRow+1) ;
ThisRowValues[NumIndicesThisRow] = 0.0;
PermutedGlobalColIndices.resize(NumIndicesThisRow+1);
PermutedGlobalColIndices[NumIndicesThisRow] = MyPermutedGlobalRowElements[i] ;
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< "NumIndicesThisRow = " << NumIndicesThisRow
<< "ThisRowValues[NumIndicesThisRow = " << ThisRowValues[NumIndicesThisRow]
<< " PermutedGlobalColIndices[NumIndcesThisRow] = " << PermutedGlobalColIndices[NumIndicesThisRow]
<< std::endl ;
#endif
NumIndicesThisRow++ ;
}
assert( Out->InsertGlobalValues( MyPermutedGlobalRowElements[i],
NumIndicesThisRow,
&ThisRowValues[0],
&PermutedGlobalColIndices[0] ) >= 0 );
}
//
Epetra_LocalMap ReplicatedMap( NumGlobalRangeElements, 0, In.Comm() );
RCP<Epetra_Map> OutRangeMap ;
RCP<Epetra_Map> OutDomainMap ;
switch( RangeMapType ) {
case 0:
OutRangeMap = PermutedRangeMap ;
break;
case 1:
OutRangeMap = rcp(&SerialRangeMap, false);
break;
case 2:
OutRangeMap = rcp(&BizarreRangeMap, false);
break;
case 3:
OutRangeMap = rcp(&ReplicatedMap, false);
break;
}
// switch( DomainMapType ) {
switch( DomainMapType ) {
case 0:
OutDomainMap = PermutedDomainMap ;
break;
case 1:
OutDomainMap = rcp(&SerialDomainMap, false);
break;
case 2:
OutDomainMap = rcp(&BizarreDomainMap, false);
break;
case 3:
OutDomainMap = rcp(&ReplicatedMap, false);
break;
}
#if 0
assert(Out->FillComplete( *PermutedDomainMap, *PermutedRangeMap )==0);
#else
assert(Out->FillComplete( *OutDomainMap, *OutRangeMap )==0);
#endif
#if 0
std::cout << __FILE__ << "::" << __LINE__ << std::endl ;
Out->Print( std::cout ) ;
#endif
return Out;
}
int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix,
int NumSameIDs,
int NumRemoteIDs,
const int * RemoteLIDs,
int NumPermuteIDs,
const int *PermuteToLIDs,
const int *PermuteFromLIDs,
int LenImports,
char* Imports,
int TargetNumRows,
int TargetNumNonzeros,
int * CSR_rowptr,
int_type * CSR_colind,
double * CSR_vals,
const std::vector<int> &SourcePids,
std::vector<int> &TargetPids)
{
// What we really need to know is where in the CSR arrays each row should start (aka the rowptr).
// We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and
// (c) Having each row copied into the right colind / values locations.
// From Epetra_CrsMatrix UnpackAndCombine():
// Each segment of Exports will be filled by a packed row of information for each row as follows:
// 1st int: GRID of row where GRID is the global row ID for the source matrix
// next int: NumEntries, Number of indices in row.
// next NumEntries: The actual indices for the row.
int i,j,rv;
int N = TargetNumRows;
int mynnz = TargetNumNonzeros;
int MyPID = SourceMatrix.Comm().MyPID();
int SizeofIntType = sizeof(int_type);
// Zero the rowptr
for(i=0; i<N+1; i++) CSR_rowptr[i]=0;
// SameIDs: Always first, always in the same place
for(i=0; i<NumSameIDs; i++)
CSR_rowptr[i]=SourceMatrix.NumMyEntries(i);
// PermuteIDs: Still local, but reordered
for(i=0; i<NumPermuteIDs; i++)
CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]);
// RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
for(i=0; i<NumRemoteIDs; i++) {
CSR_rowptr[RemoteLIDs[i]] += NumEntries;
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
}
}
}
// If multiple procs contribute to a row;
std::vector<int> NewStartRow(N+1);
// Turn row length into a real CSR_rowptr
int last_len = CSR_rowptr[0];
CSR_rowptr[0] = 0;
for(i=1; i<N+1; i++){
int new_len = CSR_rowptr[i];
CSR_rowptr[i] = last_len + CSR_rowptr[i-1];
NewStartRow[i] = CSR_rowptr[i];
last_len = new_len;
}
// Preseed TargetPids with -1 for local
if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz);
TargetPids.assign(mynnz,-1);
// Grab pointers for SourceMatrix
int * Source_rowptr, * Source_colind;
double * Source_vals;
rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals);
if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers");
// SameIDs: Copy the data over
for(i=0; i<NumSameIDs; i++) {
int FromRow = Source_rowptr[i];
int ToRow = CSR_rowptr[i];
NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i];
for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// PermuteIDs: Copy the data over
for(i=0; i<NumPermuteIDs; i++) {
int FromLID = PermuteFromLIDs[i];
int FromRow = Source_rowptr[FromLID];
int ToRow = CSR_rowptr[PermuteToLIDs[i]];
NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID];
for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// RemoteIDs: Loop structure following UnpackAndCombine
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
double* valptr = dintptr + IntSize;
for (i=0; i<NumRemoteIDs; i++) {
int ToLID = RemoteLIDs[i];
int StartRow = NewStartRow[ToLID];
NewStartRow[ToLID]+=NumEntries;
double * values = valptr;
int_type * Indices = intptr + 2;
for(j=0; j<NumEntries; j++){
CSR_vals[StartRow + j] = values[j];
CSR_colind[StartRow + j] = Indices[2*j];
TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1;
}
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
valptr = dintptr + IntSize;
}
}
}
return 0;
}
int main(int argc, char *argv[])
{
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
int numprocs = comm.NumProc();
int localproc = comm.MyPID();
///////////////////////////////////////////////////
//First figure out whether verbose output is on.
//(and if so, only turn it on for proc 0)
bool verbose = false;
if (localproc == 0) {
verbose = argument_is_present("-v", argc, argv);
}
///////////////////////////////////////////////////
int local_n = 30;
int global_n = numprocs*local_n;
Epetra_Map emap(global_n, 0, comm);
Epetra_CrsMatrix* A = create_and_fill_crs_matrix(emap);
Epetra_Vector x(emap), b(emap);
x.PutScalar(1.0);
A->Multiply(false, x, b);
x.PutScalar(0.0);
double initial_norm = resid2norm(*A, x, b);
if (verbose) {
cout << "Initial 2-norm of b-A*x: "<<initial_norm<<endl;
}
int err = test_azoo_as_precond_op(*A, x, b, verbose);
if (err != 0) {
cout << "test_azoo_as_precond_op err, test FAILED."<<endl;
return(err);
}
err = test_azoo_conv_anorm(*A, x, b, verbose);
if (err != 0) {
cout << "test_azoo_conv_anorm err, test FAILED."<<endl;
return(err);
}
if (verbose) {
cout << "testing AztecOO with AZ_conv = AZ_Anorm, and AZ_scaling = AZ_sym_diag" << endl;
}
err = test_azoo_conv_with_scaling(AZ_Anorm, AZ_sym_diag, A->Comm(), verbose);
if (err != 0) {
cout << "test_azoo_conv_with_scaling err, test FAILED."<<endl;
return(err);
}
if (verbose) {
cout << "testing AztecOO with AZ_conv = AZ_rhs, and AZ_scaling = AZ_sym_diag" << endl;
}
err = test_azoo_conv_with_scaling(AZ_rhs, AZ_sym_diag, A->Comm(), verbose);
if (err != 0) {
cout << "test_azoo_conv_with_scaling err, test FAILED."<<endl;
return(err);
}
err = test_azoo_with_ilut(*A, x, b, verbose);
if (err != 0) {
cout << "test_azoo_with_ilut err, test FAILED."<<endl;
return(err);
}
err = test_azoo_scaling(*A, x, b, verbose);
if (err != 0) {
cout << "test_azoo_scaling err="<<err<<", test FAILED."<<endl;
return(err);
}
delete A;
int* options = new int[AZ_OPTIONS_SIZE];
options[AZ_solver] = AZ_cg;
options[AZ_subdomain_solve] = AZ_none;
options[AZ_precond] = AZ_Jacobi;
if (verbose)
std::cout << "about to call test_AZ_iterate_AZ_pre_calc_AZ_reuse"
<<std::endl;
err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose);
if (err != 0) {
cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl;
return(err);
}
options[AZ_solver] = AZ_cgs;
options[AZ_subdomain_solve] = AZ_icc;
options[AZ_precond] = AZ_dom_decomp;
err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose);
if (err != 0) {
cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl;
return(err);
}
options[AZ_solver] = AZ_gmres;
options[AZ_subdomain_solve] = AZ_ilut;
err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose);
if (err != 0) {
cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl;
return(err);
}
options[AZ_solver] = AZ_tfqmr;
options[AZ_subdomain_solve] = AZ_ilu;
err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose);
if (err != 0) {
cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl;
return(err);
}
options[AZ_solver] = AZ_bicgstab;
options[AZ_subdomain_solve] = AZ_rilu;
err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose);
if (err != 0) {
cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl;
return(err);
}
err = test_AZ_iterate_then_AZ_scale_f(comm, verbose);
if (err != 0) {
cout << "test_AZ_iterate_then_AZ_scale_f err, test FAILED."<<endl;
return(err);
}
delete [] options;
err = test_bug2554(comm, verbose);
if (err != 0) {
cout << "test_bug2554 err, test FAILED."<<endl;
return(err);
}
err = test_bug2890(comm, verbose);
if (err != 0) {
cout << "test_bug2890 err, test FAILED."<<endl;
return(err);
}
cout << "********* Test passed **********" << endl;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return(0);
}
// ================================================ ====== ==== ==== == =
int ML_Epetra::RefMaxwell_Aggregate_Nodes(const Epetra_CrsMatrix & A, Teuchos::ParameterList & List, ML_Comm * ml_comm, std::string PrintMsg,
ML_Aggregate_Struct *& MLAggr,ML_Operator *&P, int &NumAggregates){
/* Output level */
bool verbose, very_verbose;
int OutputLevel = List.get("ML output", -47);
if(OutputLevel == -47) OutputLevel = List.get("output", 1);
if(OutputLevel>=15) very_verbose=verbose=true;
if(OutputLevel > 5) {very_verbose=false;verbose=true;}
else very_verbose=verbose=false;
/* Wrap A in a ML_Operator */
ML_Operator* A_ML = ML_Operator_Create(ml_comm);
ML_Operator_WrapEpetraCrsMatrix(const_cast<Epetra_CrsMatrix*>(&A),A_ML);
/* Pull Teuchos Options */
std::string CoarsenType = List.get("aggregation: type", "Uncoupled");
double Threshold = List.get("aggregation: threshold", 0.0);
int NodesPerAggr = List.get("aggregation: nodes per aggregate",
ML_Aggregate_Get_OptimalNumberOfNodesPerAggregate());
bool UseAux = List.get("aggregation: aux: enable",false);
double AuxThreshold = List.get("aggregation: aux: threshold",0.0);
int MaxAuxLevels = List.get("aggregation: aux: max levels",10);
ML_Aggregate_Create(&MLAggr);
ML_Aggregate_Set_MaxLevels(MLAggr, 2);
ML_Aggregate_Set_StartLevel(MLAggr, 0);
ML_Aggregate_Set_Threshold(MLAggr, Threshold);
ML_Aggregate_Set_MaxCoarseSize(MLAggr,1);
MLAggr->cur_level = 0;
ML_Aggregate_Set_Reuse(MLAggr);
MLAggr->keep_agg_information = 1;
P = ML_Operator_Create(ml_comm);
/* Process Teuchos Options */
if (CoarsenType == "Uncoupled")
ML_Aggregate_Set_CoarsenScheme_Uncoupled(MLAggr);
else if (CoarsenType == "Uncoupled-MIS"){
ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr);
}
else if (CoarsenType == "METIS"){
ML_Aggregate_Set_CoarsenScheme_METIS(MLAggr);
ML_Aggregate_Set_NodesPerAggr(0, MLAggr, 0, NodesPerAggr);
}/*end if*/
else {
if(!A.Comm().MyPID()) printf("%s Unsupported (1,1) block aggregation type(%s), resetting to uncoupled-mis\n",PrintMsg.c_str(),CoarsenType.c_str());
ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr);
}
/* Setup Aux Data */
if(UseAux) {
A_ML->aux_data->enable=1;
A_ML->aux_data->threshold=AuxThreshold;
A_ML->aux_data->max_level=MaxAuxLevels;
ML_Init_Aux(A_ML,List);
if(verbose && !A.Comm().MyPID()) {
printf("%s Using auxiliary matrix\n",PrintMsg.c_str());
printf("%s aux threshold = %e\n",PrintMsg.c_str(),A_ML->aux_data->threshold);
}
}
/* Aggregate Nodes */
int printlevel=ML_Get_PrintLevel();
if(verbose) ML_Set_PrintLevel(10);
NumAggregates = ML_Aggregate_Coarsen(MLAggr,A_ML, &P, ml_comm);
if(verbose) ML_Set_PrintLevel(printlevel);
if (NumAggregates == 0){
std::cerr << "Found 0 aggregates, perhaps the problem is too small." << std::endl;
ML_CHK_ERR(-2);
}/*end if*/
else if(very_verbose) printf("[%d] %s %d aggregates created invec_leng=%d\n",A.Comm().MyPID(),PrintMsg.c_str(),NumAggregates,P->invec_leng);
if(verbose){
int globalAggs=0;
A.Comm().SumAll(&NumAggregates,&globalAggs,1);
if(!A.Comm().MyPID()) {
printf("%s Aggregation threshold = %e\n",PrintMsg.c_str(),Threshold);
printf("%s Global aggregates = %d\n",PrintMsg.c_str(),globalAggs);
}
}
/* Cleanup */
ML_qr_fix_Destroy();
if(UseAux) ML_Finalize_Aux(A_ML);
ML_Operator_Destroy(&A_ML);
return 0;
}
