//=======================================================
int EpetraExt_HypreIJMatrix::RightScale(const Epetra_Vector& X) {
// First we need to import off-processor values of the vector
Epetra_Import Importer(RowMatrixColMap(), RowMatrixRowMap());
Epetra_Vector Import_Vector(RowMatrixColMap(), true);
EPETRA_CHK_ERR(Import_Vector.Import(X, Importer, Insert, 0));
for(int i = 0; i < NumMyRows_; i++){
//Vector-scalar mult on ith column
int num_entries;
double *values;
int *indices;
// Get values and indices of ith row of matrix
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixGetRow(ParMatrix_,i+MyRowStart_, &num_entries, &indices, &values));
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixRestoreRow(ParMatrix_,i+MyRowStart_,&num_entries,&indices,&values));
Teuchos::Array<int> new_indices; new_indices.resize(num_entries);
Teuchos::Array<double> new_values; new_values.resize(num_entries);
for(int j = 0; j < num_entries; j++){
// Multiply column j with jth element
int index = RowMatrixColMap().LID(indices[j]);
TEUCHOS_TEST_FOR_EXCEPTION(index < 0, std::logic_error, "Index is negtive.");
new_values[j] = values[j] * Import_Vector[index];
new_indices[j] = indices[j];
}
// Finally set values of the Matrix for this row
int rows[1];
rows[0] = i+MyRowStart_;
EPETRA_CHK_ERR(HYPRE_IJMatrixSetValues(Matrix_, 1, &num_entries, rows, &new_indices[0], &new_values[0]));
}
HaveNumericConstants_ = false;
UpdateFlops(NumGlobalNonzeros());
return 0;
} //RightScale()
void Algorithm::run()
{
emit wasStarted();
splitter->split();
const QList<Package>& packages = splitter->getSamples();
std::for_each(packages.begin(), packages.end(), Importer());
emit wasFinished(this);
}
void HeqProblem::ComputeJacobian(const Epetra_Vector & x, const double c, double alpha)
{
// Set up variables
double *Values = new double[N]; // holds the values computed for each row
int *Indices = new int[N]; // holds the column index corresponding to each calculated value
int ilocal, iglobal, j; // counter variables used for loops:
// ilocal = counter for local elements on this processor;
// iglobal = counter to signify global position across all procs
int Myele; // holds the number of elements on the processor
double temp; // temporary variable used for calculations
// Set up the importer for the overlap map
Epetra_Import Importer(OverlapMap, Matrix->Map());
// Define the overlap map
Epetra_Vector overlap_x(OverlapMap);
// Fill in the overlap map
overlap_x.Import(x, Importer, Insert);
// Get the no. of elements on this processor
Myele = Map.NumMyElements();
// Compute the values for the Jacobian matrix
// Loop over the number of elements on the local processor
for (ilocal=0; ilocal<Myele; ilocal++) {
iglobal=Map.GID(ilocal); // Get the global ID for the local element
for (j=0; j<N; j++) { // compute each Jacobian element
temp = pow(hold[iglobal],2.0);
temp = -temp*(c/(double)(2*N))*((double)iglobal+0.5)/(double)(iglobal+j+1);
if (iglobal==j)
temp = temp + 1.0;
Values[j] = alpha*temp; // save the computed value
Indices[j] = j; // assign the column index of the computed value
} // end of j loop
// Insert the values calculated into the matrix by row
if (FirstTime == true)
Matrix->InsertGlobalValues(iglobal, N, &Values[0], &Indices[0]);
else
Matrix->ReplaceGlobalValues(iglobal, N, &Values[0], &Indices[0]);
} // end of ilocal loop
// Done with matrix calculations!
// Do final steps to deallocate memory and finish the matrix fill.
if (FirstTime == true)
Matrix->FillComplete();
delete [] Indices;
delete [] Values;
FirstTime = false;
} // end of ComputeJacobian
bool
CrsMatrix_Dirichlet::
fwd()
{
Epetra_CrsMatrix & Matrix = *origObj_;
const Epetra_Map & RowMap = Matrix.RowMap();
const Epetra_Map & ColMap = Matrix.ColMap();
int NumMyElements = RowMap.NumMyElements();
int NumMyColElements = ColMap.NumMyElements();
if( symmetric_ && colSet_.empty() ) // get non-local column info
{
if( Matrix.IndicesAreGlobal() )
{
Epetra_Import Importer( ColMap, RowMap );
Epetra_IntVector colLocations( ColMap );
colLocations.Import( locations_, Importer, Insert );
for( int i = 0; i < NumMyColElements; ++ i )
if( colLocations[i] ) colSet_.insert(i);
}
else
{
for( int i = 0; i < NumMyElements; ++i )
if( locations_[i] ) colSet_.insert(i);
}
}
for( int i = 0; i < NumMyElements; ++i )
{
int * Indices;
double * Vals;
int NumIndices;
if( locations_[i] ) //this is a Dirichlet BC location
{
Matrix.ExtractMyRowView( i, NumIndices, Vals, Indices );
for( int j = 0; j < NumIndices; ++j )
{
if( Indices[j] == i ) Vals[i] = 1.0;
else Vals[i] = 0.0;
}
}
else if( symmetric_ )
{
Matrix.ExtractMyRowView( i, NumIndices, Vals, Indices );
for( int j = 0; j < NumIndices; ++j )
if( colSet_.count( Indices[j] ) ) Vals[j] = 0.0;
}
}
return true;
}
//
// Compute the value of F(x)
//
void HeqProblem::ComputeF(const Epetra_Vector & x, Epetra_Vector & f, const double c)
{
// Set up variables
int ilocal, j, iglobal; // counter variables used for loops:
// ilocal = counter for local elements on this processor;
// iglobal = counter to signify global position across all procs
int Myele; // holds the number of elements on the processor
double temp; // temporary variable used for calculations
hold = new double[N]; // array to hold calculated values used to compute F(x) and
// the Jacobian matrix
// Set up the importer for the overlap map
Epetra_Import Importer(OverlapMap, Matrix->Map());
// Define the overlap map
Epetra_Vector overlap_x(OverlapMap);
// Fill in the overlap map
overlap_x.Import(x, Importer, Insert);
// Get the no. of elements on this processor
Myele = Map.NumMyElements();
// Compute the function values
// Loop over the number of elements on the local processor
for (ilocal=0; ilocal<Myele; ilocal++) {
temp=0.0;
iglobal=Map.GID(ilocal); // Get the global ID for the local element
for (j=0; j<N; j++) { // Compute the value of Sum_{j=1}^N { u_i*x_j / (u_i+u_j) }
temp = temp + ((double)iglobal+0.5)*overlap_x[j]/(double)(iglobal+j+1);
}
temp = 1.0 - c*temp/(double)(2*N);
// Store the value of [1-c/2N * Sum()]^{-1} for use in computing the Jacobian
hold[iglobal] = pow(temp,-1.0);
f[ilocal] = overlap_x[iglobal]-hold[iglobal]; // compute final F(x)_i value
} // end of ilocal loop
} // end computeF
CrsGraph_AMD::NewTypeRef
CrsGraph_AMD::
operator()( OriginalTypeRef orig )
{
origObj_ = &orig;
int n = orig.NumMyRows();
int nnz = orig.NumMyNonzeros();
//create std CRS format
std::vector<int> ia(n+1,0);
std::vector<int> ja(nnz);
int cnt;
for( int i = 0; i < n; ++i )
{
int * tmpP = &ja[ia[i]];
orig.ExtractMyRowCopy( i, nnz-ia[i], cnt, tmpP );
ia[i+1] = ia[i] + cnt;
}
//trim down to local only
std::vector<int> iat(n+1);
std::vector<int> jat(nnz);
int loc = 0;
for( int i = 0; i < n; ++i )
{
iat[i] = loc;
for( int j = ia[i]; j < ia[i+1]; ++j )
{
if( ja[j] < n )
jat[loc++] = ja[j];
else
break;
}
}
iat[n] = loc;
if( verbose_ )
{
std::cout << "Orig Graph\n";
std::cout << orig << std::endl;
std::cout << "-----------------------------------------\n";
std::cout << "CRS Format Graph\n";
std::cout << "-----------------------------------------\n";
for( int i = 0; i < n; ++i )
{
std::cout << i << ": " << iat[i+1] << ": ";
for( int j = iat[i]; j<iat[i+1]; ++j )
std::cout << " " << jat[j];
std::cout << std::endl;
}
std::cout << "-----------------------------------------\n";
}
std::vector<int> perm(n);
std::vector<double> info(AMD_INFO);
amd_order( n, &iat[0], &jat[0], &perm[0], NULL, &info[0] );
if( info[AMD_STATUS] == AMD_INVALID )
std::cout << "AMD ORDERING: Invalid!!!!\n";
if( verbose_ )
{
std::cout << "-----------------------------------------\n";
std::cout << "AMD Output\n";
std::cout << "-----------------------------------------\n";
std::cout << "STATUS: " << info[AMD_STATUS] << std::endl;
std::cout << "SYMM: " << info[AMD_SYMMETRY] << std::endl;
std::cout << "N: " << info[AMD_N] << std::endl;
std::cout << "NZ: " << info[AMD_NZ] << std::endl;
std::cout << "SYMM: " << info[AMD_SYMMETRY] << std::endl;
std::cout << "NZDIAG: " << info[AMD_NZDIAG] << std::endl;
std::cout << "NZ A+At: " << info[AMD_NZ_A_PLUS_AT] << std::endl;
std::cout << "NDENSE: " << info[AMD_SYMMETRY] << std::endl;
std::cout << "Perm\n";
for( int i = 0; i<n; ++i )
std::cout << perm[i] << std::endl;
std::cout << "-----------------------------------------\n";
}
//Generate New Domain and Range Maps
//for now, assume they start out as identical
const Epetra_BlockMap & OldMap = orig.RowMap();
int nG = orig.NumGlobalRows();
std::vector<int> newElements( n );
for( int i = 0; i < n; ++i )
newElements[i] = OldMap.GID( perm[i] );
NewMap_ = new Epetra_Map( nG, n, &newElements[0], OldMap.IndexBase(), OldMap.Comm() );
if( verbose_ )
{
std::cout << "Old Map\n";
std::cout << OldMap << std::endl;
std::cout << "New Map\n";
std::cout << *NewMap_ << std::endl;
}
//Generate New Graph
NewGraph_ = new Epetra_CrsGraph( Copy, *NewMap_, 0 );
Epetra_Import Importer( *NewMap_, OldMap );
NewGraph_->Import( orig, Importer, Insert );
NewGraph_->FillComplete();
if( verbose_ )
{
std::cout << "New CrsGraph\n";
std::cout << *NewGraph_ << std::endl;
}
newObj_ = NewGraph_;
return *NewGraph_;
}
//=============================================================================
int Epetra_FastCrsMatrix::Multiply(bool TransA, const Epetra_MultiVector& X, Epetra_MultiVector& Y) const {
//
// This function forms the product Y = A * Y or Y = A' * X
//
if (X.NumVectors()==1 && Y.NumVectors()==1) {
double * xp = (double *) X[0];
double * yp = (double *) Y[0];
Epetra_Vector x(View, X.Map(), xp);
Epetra_Vector y(View, Y.Map(), yp);
return(Multiply(TransA, x, y));
}
if (!Filled()) EPETRA_CHK_ERR(-1); // Matrix must be filled.
int i, j, k;
int * NumEntriesPerRow = NumEntriesPerRow_;
int ** Indices = Indices_;
double ** Values = Values_;
double **Xp = (double**)X.Pointers();
double **Yp = (double**)Y.Pointers();
int NumVectors = X.NumVectors();
int NumMyCols_ = NumMyCols();
// Need to better manage the Import and Export vectors:
// - Need accessor functions
// - Need to make the NumVector match (use a View to do this)
// - Need to look at RightScale and ColSum routines too.
if (!TransA) {
// If we have a non-trivial importer, we must import elements that are permuted or are on other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),NumVectors); // Create import vector if needed
ImportVector_->Import(X, *Importer(), Insert);
Xp = (double**)ImportVector_->Pointers();
}
// If we have a non-trivial exporter, we must export elements that are permuted or belong to other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=NumVectors) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),NumVectors); // Create Export vector if needed
Yp = (double**)ExportVector_->Pointers();
}
// Do actual computation
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (k=0; k<NumVectors; k++) {
double sum = 0.0;
for (j=0; j < NumEntries; j++) sum += RowValues[j] * Xp[k][RowIndices[j]];
Yp[k][i] = sum;
}
}
if (Exporter()!=0) Y.Export(*ExportVector_, *Exporter(), Add); // Fill Y with Values from export vector
}
else { // Transpose operation
// If we have a non-trivial exporter, we must import elements that are permuted or are on other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=NumVectors) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),NumVectors); // Create Export vector if needed
ExportVector_->Import(X, *Exporter(), Insert);
Xp = (double**)ExportVector_->Pointers();
}
// If we have a non-trivial importer, we must export elements that are permuted or belong to other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),NumVectors); // Create import vector if needed
Yp = (double**)ImportVector_->Pointers();
}
// Do actual computation
for (k=0; k<NumVectors; k++)
for (i=0; i < NumMyCols_; i++) Yp[k][i] = 0.0; // Initialize y for transpose multiply
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (k=0; k<NumVectors; k++) {
for (j=0; j < NumEntries; j++) Yp[k][RowIndices[j]] += RowValues[j] * Xp[k][i];
}
}
if (Importer()!=0) Y.Export(*ImportVector_, *Importer(), Add); // Fill Y with Values from export vector
}
UpdateFlops(2*NumVectors*NumGlobalNonzeros64());
return(0);
}
//=============================================================================
int Epetra_FastCrsMatrix::Multiply(bool TransA, const Epetra_Vector& x, Epetra_Vector& y) const {
//
// This function forms the product y = A * x or y = A' * x
//
int i, j;
double * xp = (double*)x.Values();
double *yp = (double*)y.Values();
int NumMyCols_ = NumMyCols();
if (!TransA) {
// If we have a non-trivial importer, we must import elements that are permuted or are on other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=1) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),1); // Create import vector if needed
ImportVector_->Import(x, *Importer(), Insert);
xp = (double*)ImportVector_->Values();
}
// If we have a non-trivial exporter, we must export elements that are permuted or belong to other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=1) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),1); // Create Export vector if needed
yp = (double*)ExportVector_->Values();
}
// Do actual computation
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
double sum = 0.0;
for (j=0; j < NumEntries; j++) sum += RowValues[j] * xp[RowIndices[j]];
yp[i] = sum;
}
if (Exporter()!=0) y.Export(*ExportVector_, *Exporter(), Add); // Fill y with Values from export vector
}
else { // Transpose operation
// If we have a non-trivial exporter, we must import elements that are permuted or are on other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=1) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),1); // Create Export vector if needed
ExportVector_->Import(x, *Exporter(), Insert);
xp = (double*)ExportVector_->Values();
}
// If we have a non-trivial importer, we must export elements that are permuted or belong to other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=1) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),1); // Create import vector if needed
yp = (double*)ImportVector_->Values();
}
// Do actual computation
for (i=0; i < NumMyCols_; i++) yp[i] = 0.0; // Initialize y for transpose multiply
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (j=0; j < NumEntries; j++) yp[RowIndices[j]] += RowValues[j] * xp[i];
}
if (Importer()!=0) y.Export(*ImportVector_, *Importer(), Add); // Fill y with Values from export vector
}
UpdateFlops(2*NumGlobalNonzeros64());
return(0);
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
//=============================================================================
// If FirstTime is true, then build SerialMap and ImportToSerial,
// otherwise simply re-ship the matrix, so that the numerical values
// are updated.
int Amesos_Umfpack::ConvertToSerial(const bool FirstTime)
{
ResetTimer(0);
ResetTimer(1);
const Epetra_Map &OriginalMap = Matrix()->RowMatrixRowMap() ;
NumGlobalElements_ = Matrix()->NumGlobalRows();
numentries_ = Matrix()->NumGlobalNonzeros();
assert (NumGlobalElements_ == Matrix()->NumGlobalCols());
int NumMyElements_ = 0 ;
if (MyPID_ == 0) NumMyElements_ = NumGlobalElements_;
IsLocal_ = ( OriginalMap.NumMyElements() ==
OriginalMap.NumGlobalElements() )?1:0;
// if ( AddZeroToDiag_ ) IsLocal_ = 0 ; // bug # Umfpack does not support AddZeroToDiag_
Comm().Broadcast( &IsLocal_, 1, 0 ) ;
// Convert Original Matrix to Serial (if it is not already)
//
if (IsLocal_== 1) {
SerialMatrix_ = Matrix();
}
else
{
if (FirstTime)
{
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_,NumMyElements_,
0,Comm()));
if (SerialMap_.get() == 0)
AMESOS_CHK_ERR(-1);
ImportToSerial_ = rcp(new Epetra_Import (SerialMap(),OriginalMap));
if (ImportToSerial_.get() == 0)
AMESOS_CHK_ERR(-1);
}
SerialCrsMatrixA_ = rcp(new Epetra_CrsMatrix(Copy,SerialMap(),0));
if (SerialCrsMatrixA_.get() == 0)
AMESOS_CHK_ERR(-1);
SerialCrsMatrix().Import(*Matrix(), Importer(),Insert);
#if 0
I was not able to make this work - 11 Feb 2006
if (AddZeroToDiag_ ) {
int OriginalTracebackMode = SerialCrsMatrix().GetTracebackMode() ;
SerialCrsMatrix().SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) ) ; // ExportToSerial is called both by PerformSymbolicFactorization() and PerformNumericFactorization(). When called by the latter, the call to insertglobalvalues is both unnecessary and illegal. Fortunately, Epetra allows us to ignore the error message by setting the traceback mode to 0.
//
// Add 0.0 to each diagonal entry to avoid empty diagonal entries;
//
double zero = 0.0;
for ( int i = 0 ; i < SerialMap_->NumGlobalElements(); i++ )
if ( SerialCrsMatrix().LRID(i) >= 0 )
SerialCrsMatrix().InsertGlobalValues( i, 1, &zero, &i ) ;
SerialCrsMatrix().SetTracebackMode( OriginalTracebackMode ) ;
}
#endif
SerialCrsMatrix().FillComplete();
SerialMatrix_ = &SerialCrsMatrix();
assert( numentries_ == SerialMatrix_->NumGlobalNonzeros()); // This should be set to an assignment if AddToZeroDiag is non -zero
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
return(0);
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Tpetra::MpiComm<OrdinalType, ScalarType> Comm(MPI_COMM_WORLD);
#else
Tpetra::SerialComm<OrdinalType, ScalarType> Comm;
#endif
// Get zero and one for the OrdinalType
OrdinalType const OrdinalZero = Teuchos::ScalarTraits<OrdinalType>::zero();
OrdinalType const OrdinalOne = Teuchos::ScalarTraits<OrdinalType>::one();
// Get zero and one for the ScalarType
ScalarType const ScalarZero = Teuchos::ScalarTraits<ScalarType>::zero();
ScalarType const ScalarOne = Teuchos::ScalarTraits<ScalarType>::one();
// Creates a vector of size `length', then set the elements values.
OrdinalType length = OrdinalOne * 2 * Comm.getNumImages();
OrdinalType indexBase = OrdinalZero;
// 1) Creation of a platform
#ifdef HAVE_MPI
const Tpetra::MpiPlatform <OrdinalType, OrdinalType> platformE(MPI_COMM_WORLD);
const Tpetra::MpiPlatform <OrdinalType, ScalarType> platformV(MPI_COMM_WORLD);
#else
const Tpetra::SerialPlatform <OrdinalType, OrdinalType> platformE;
const Tpetra::SerialPlatform <OrdinalType, ScalarType> platformV;
#endif
// linearly distributed space
Tpetra::ElementSpace<OrdinalType> LinearSpace(length, indexBase, platformE);
Tpetra::VectorSpace<OrdinalType, ScalarType> VectorLinearSpace(LinearSpace, platformV);
// all elements on processor 0
OrdinalType NumMyElements = (Comm.getMyImageID() == 0)?length:0;
Tpetra::ElementSpace<OrdinalType> ZeroSpace(-1, NumMyElements, indexBase, platformE);
Tpetra::VectorSpace<OrdinalType, ScalarType> VectorZeroSpace(ZeroSpace, platformV);
Tpetra::Vector<OrdinalType, ScalarType> LinearV(VectorLinearSpace);
Tpetra::Vector<OrdinalType, ScalarType> ZeroV(VectorZeroSpace);
LinearV.setAllToScalar(Comm.getMyImageID());
cout << LinearV;
Tpetra::Import<OrdinalType> Importer(LinearSpace, ZeroSpace);
ZeroV.doImport(LinearV, Importer, Tpetra::Insert);
cout << ZeroV;
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return(EXIT_SUCCESS);
}
CrsGraph_SymmRCM::NewTypeRef
CrsGraph_SymmRCM::
operator()( CrsGraph_SymmRCM::OriginalTypeRef orig )
{
origObj_ = &orig;
int err;
//Generate Local Transpose Graph
CrsGraph_Transpose transposeTransform;
Epetra_CrsGraph & trans = transposeTransform( orig );
//Generate Local Symmetric Adj. List
//Find Min Degree Node While at it
int NumNodes = orig.NumMyRows();
int * LocalRow;
int * LocalRowTrans;
int RowSize, RowSizeTrans;
std::vector< std::vector<int> > AdjList( NumNodes );
int MinDegree = NumNodes;
int MinDegreeNode;
for( int i = 0; i < NumNodes; ++i )
{
orig.ExtractMyRowView( i, RowSize, LocalRow );
trans.ExtractMyRowView( i, RowSizeTrans, LocalRowTrans );
std::set<int> adjSet;
for( int j = 0; j < RowSize; ++j )
if( LocalRow[j] < NumNodes ) adjSet.insert( LocalRow[j] );
for( int j = 0; j < RowSizeTrans; ++j )
if( LocalRowTrans[j] < NumNodes ) adjSet.insert( LocalRowTrans[j] );
std::set<int>::iterator iterS = adjSet.begin();
std::set<int>::iterator endS = adjSet.end();
AdjList[i].resize( adjSet.size() );
for( int j = 0; iterS != endS; ++iterS, ++j ) AdjList[i][j] = *iterS;
if( AdjList[i].size() < MinDegree )
{
MinDegree = AdjList[i].size();
MinDegreeNode = i;
}
}
BFT * BestBFT;
bool TooWide;
//std::cout << "SymmRCM::bruteForce_ : " << bruteForce_ << std::endl;
if( bruteForce_ )
{
int bestWidth = NumNodes;
int bestDepth = 0;
for( int i = 0; i < NumNodes; ++i )
{
BFT * TestBFT = new BFT( AdjList, i, NumNodes, TooWide );
if( TestBFT->Depth() > bestDepth ||
( TestBFT->Depth() == bestDepth && TestBFT->Width() < bestWidth ) )
{
BestBFT = TestBFT;
bestDepth = TestBFT->Depth();
bestWidth = TestBFT->Width();
}
else
delete TestBFT;
}
}
else
{
//Construct BFT for first
BestBFT = new BFT( AdjList, MinDegreeNode, NumNodes, TooWide );
int MinWidth = BestBFT->Width();
int BestWidth = MinWidth;
int Diameter = BestBFT->Depth();
std::vector<int> Leaves;
BestBFT->NonNeighborLeaves( Leaves, AdjList, testLeafWidth_ );
bool DeeperFound;
bool NarrowerFound;
bool Finished = false;
while( !Finished )
{
DeeperFound = false;
NarrowerFound = false;
for( int i = 0; i < Leaves.size(); ++i )
{
BFT * TestBFT = new BFT( AdjList, Leaves[i], MinWidth, TooWide );
if( TooWide )
delete TestBFT;
else
{
if( TestBFT->Width() < MinWidth ) MinWidth = TestBFT->Width();
if( TestBFT->Depth() > Diameter )
{
delete BestBFT;
Diameter = TestBFT->Depth();
BestWidth = TestBFT->Width();
BestBFT = TestBFT;
DeeperFound = true;
NarrowerFound = false;
}
else if( (TestBFT->Depth()==Diameter) && (TestBFT->Width()<BestWidth) )
{
delete BestBFT;
BestWidth = TestBFT->Width();
BestBFT = TestBFT;
NarrowerFound = true;
}
else delete TestBFT;
}
}
if( DeeperFound )
BestBFT->NonNeighborLeaves( Leaves, AdjList, testLeafWidth_ );
else if( NarrowerFound )
Finished = true;
else Finished = true;
}
}
//std::cout << "\nSymmRCM:\n";
//std::cout << "----------------------------\n";
//std::cout << " Depth: " << BestBFT->Depth() << std::endl;
//std::cout << " Width: " << BestBFT->Width() << std::endl;
//std::cout << "----------------------------\n\n";
std::vector<int> RCM;
BestBFT->ReverseVector( RCM );
for( int i = 0; i < NumNodes; ++i )
RCM[i] = orig.RowMap().GID( RCM[i] );
//Generate New Row Map
RCMMap_ = new Epetra_Map( orig.RowMap().NumGlobalElements(),
NumNodes,
&RCM[0],
orig.RowMap().IndexBase(),
orig.RowMap().Comm() );
//Generate New Col Map
if( RCMMap_->DistributedGlobal() )
{
std::vector<int> colIndices = RCM;
const Epetra_BlockMap & origColMap = orig.ColMap();
if( origColMap.NumMyElements() > RCMMap_->NumMyElements() )
{
for( int i = RCMMap_->NumMyElements(); i < origColMap.NumMyElements(); ++i )
colIndices.push_back( origColMap.GID(i) );
}
RCMColMap_ = new Epetra_Map( orig.ColMap().NumGlobalElements(),
colIndices.size(),
&colIndices[0],
orig.ColMap().IndexBase(),
orig.ColMap().Comm() );
}
else
RCMColMap_ = RCMMap_;
//Create New Graph
Epetra_Import Importer( *RCMMap_, orig.RowMap() );
Epetra_CrsGraph * RCMGraph = new Epetra_CrsGraph( Copy, *RCMMap_, *RCMColMap_, 0 );
RCMGraph->Import( orig, Importer, Insert );
RCMGraph->FillComplete();
/*
std::cout << "origGraph\n";
std::cout << orig;
std::cout << "RCMGraph\n";
std::cout << *RCMGraph;
*/
newObj_ = RCMGraph;
return *RCMGraph;
}
//=============================================================================
int Amesos_Dscpack::PerformNumericFactorization()
{
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix* RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ;
int numrows = RowMatrixA->NumGlobalRows();
assert( numrows == RowMatrixA->NumGlobalCols() );
//
// Call Dscpack to perform Numeric Factorization
//
std::vector<double> MyANonZ;
#if 0
if ( IsNumericFactorizationOK_ ) {
DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject);
}
#endif
DscRowMap_ = Teuchos::rcp(new Epetra_Map(numrows, NumLocalCols,
LocalStructOldNum, 0, Comm()));
if (DscRowMap_.get() == 0)
AMESOS_CHK_ERR(-1);
Importer_ = rcp(new Epetra_Import(DscRowMap(), OriginalMap));
//
// Import from the CrsMatrix
//
Epetra_CrsMatrix DscMat(Copy, DscRowMap(), 0);
AMESOS_CHK_ERR(DscMat.Import(*RowMatrixA, Importer(), Insert));
AMESOS_CHK_ERR(DscMat.FillComplete());
DscColMap_ = Teuchos::rcp(new Epetra_Map(DscMat.RowMatrixColMap()));
assert( MyDscRank >= 0 || NumLocalNonz == 0 ) ;
assert( MyDscRank >= 0 || NumLocalCols == 0 ) ;
assert( MyDscRank >= 0 || NumGlobalCols == 0 ) ;
MyANonZ.resize( NumLocalNonz ) ;
int NonZIndex = 0 ;
int max_num_entries = DscMat.MaxNumEntries() ;
std::vector<int> col_indices( max_num_entries ) ;
std::vector<double> mat_values( max_num_entries ) ;
assert( NumLocalCols == DscRowMap().NumMyElements() ) ;
std::vector<int> my_global_elements( NumLocalCols ) ;
AMESOS_CHK_ERR(DscRowMap().MyGlobalElements( &my_global_elements[0] ) ) ;
std::vector<int> GlobalStructOldColNum( NumGlobalCols ) ;
typedef std::pair<int, double> Data;
std::vector<Data> sort_array(max_num_entries);
std::vector<int> sort_indices(max_num_entries);
for ( int i = 0; i < NumLocalCols ; i++ ) {
assert( my_global_elements[i] == LocalStructOldNum[i] ) ;
int num_entries_this_row;
#ifdef USE_LOCAL
AMESOS_CHK_ERR( DscMat.ExtractMyRowCopy( i, max_num_entries, num_entries_this_row,
&mat_values[0], &col_indices[0] ) ) ;
#else
AMESOS_CHK_ERR( DscMat.ExtractGlobalRowCopy( DscMat.GRID(i), max_num_entries, num_entries_this_row,
&mat_values[0], &col_indices[0] ) ) ;
#endif
int OldRowNumber = LocalStructOldNum[i] ;
if (GlobalStructOwner[ OldRowNumber ] == -1)
AMESOS_CHK_ERR(-1);
int NewRowNumber = GlobalStructNewColNum[ my_global_elements[ i ] ] ;
//
// Sort the column elements
//
for ( int j = 0; j < num_entries_this_row; j++ ) {
#ifdef USE_LOCAL
sort_array[j].first = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ;
sort_indices[j] = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ;
#else
sort_array[j].first = GlobalStructNewColNum[ col_indices[j] ];
#endif
sort_array[j].second = mat_values[j] ;
}
sort(&sort_array[0], &sort_array[num_entries_this_row]);
for ( int j = 0; j < num_entries_this_row; j++ ) {
int NewColNumber = sort_array[j].first ;
if ( NewRowNumber <= NewColNumber ) MyANonZ[ NonZIndex++ ] = sort_array[j].second ;
#ifndef USE_LOCAL
assert( NonZIndex <= NumLocalNonz ); // This assert can fail on non-symmetric matrices
#endif
}
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
const int SchemeCode = 1;
#ifndef USE_LOCAL
assert( NonZIndex == NumLocalNonz );
#endif
AMESOS_CHK_ERR( DSC_NFactor ( PrivateDscpackData_->MyDSCObject_, SchemeCode, &MyANonZ[0],
DSC_LLT, DSC_LBLAS3, DSC_DBLAS2 ) ) ;
} // if ( MyDscRank >= 0 )
IsNumericFactorizationOK_ = true ;
NumFactTime_ = AddTime("Total numeric factorization time", NumFactTime_, 0);
return(0);
}
// ============================================================================
int ML_Epetra::MultiLevelPreconditioner::
CreateAuxiliaryMatrixCrs(Epetra_FECrsMatrix* &FakeMatrix)
{
int NumMyRows = RowMatrix_->NumMyRows();
const Epetra_Map& RowMap = RowMatrix_->RowMatrixRowMap();
const Epetra_Map& ColMap = RowMatrix_->RowMatrixColMap();
FakeMatrix = new Epetra_FECrsMatrix(Copy,RowMap,
2*RowMatrix_->MaxNumEntries());
if (FakeMatrix == 0)
ML_CHK_ERR(-1); // something went wrong
int NumDimensions = 0;
double* x_coord = List_.get("x-coordinates", (double *)0);
if (x_coord != 0) ++NumDimensions;
// at least x-coordinates must be not null
if( NumDimensions == 0 ) {
std::cerr << ErrorMsg_ << "Option `aggregation: use auxiliary matrix' == true" << std::endl
<< ErrorMsg_ << "requires x-, y-, or z-coordinates." << std::endl
<< ErrorMsg_ << "You must specify them using options" << std::endl
<< ErrorMsg_ << "`x-coordinates' (and equivalently for" << std::endl
<< ErrorMsg_ << "y- and z-." << std::endl;
ML_CHK_ERR(-2); // wrong parameters
}
double* y_coord = List_.get("y-coordinates", (double *)0);
if (y_coord != 0) ++NumDimensions;
double* z_coord = List_.get("z-coordinates", (double *)0);
if (z_coord != 0) ++NumDimensions;
// small check to avoid strange behavior
if( z_coord != 0 && y_coord == 0 ) {
std::cerr << ErrorMsg_ << "Something wrong: `y-coordinates'" << std::endl
<< ErrorMsg_ << "is null, while `z-coordinates' is null" << std::endl;
ML_CHK_ERR(-3); // something went wrong
}
double theta = List_.get("aggregation: theta",0.0);
bool SymmetricPattern = List_.get("aggregation: use symmetric pattern",false);
// usual crap to clutter the output
if( verbose_ ) {
std::cout << std::endl;
std::cout << PrintMsg_ << "*** Using auxiliary matrix to create the aggregates" << std::endl
<< PrintMsg_ << "*** Number of dimensions = " << NumDimensions << std::endl
<< PrintMsg_ << "*** theta = " << theta;
if( SymmetricPattern ) std::cout << ", using symmetric pattern" << std::endl;
else std::cout << ", using original pattern" << std::endl;
std::cout << std::endl;
}
// create vectors containing coordinates, replicated for all unknonws
// FIXME: I don't really need Z in all cases
// for west
// I am over-allocating, for large number of equations per node
// this is not optimal. However, it is a only-once importing
// of some more data. It should harm too much...
//
// The following will work for constant number of equations per
// node only. The fix should be easy, though.
Epetra_Vector RowX(RowMap); RowX.PutScalar(0.0);
Epetra_Vector RowY(RowMap); RowY.PutScalar(0.0);
Epetra_Vector RowZ(RowMap); RowZ.PutScalar(0.0);
for (int i = 0 ; i < NumMyRows ; i += NumPDEEqns_) {
RowX[i] = x_coord[i / NumPDEEqns_];
if (NumDimensions > 1) RowY[i] = y_coord[i / NumPDEEqns_];
if (NumDimensions > 2) RowZ[i] = z_coord[i / NumPDEEqns_];
}
// create vectors containing coordinates for columns
// (this is useful only if MIS/ParMETIS are used)
Epetra_Vector ColX(ColMap); ColX.PutScalar(0.0);
Epetra_Vector ColY(ColMap); ColY.PutScalar(0.0);
Epetra_Vector ColZ(ColMap); ColZ.PutScalar(0.0);
// get coordinates for non-local nodes (in column map)
Epetra_Import Importer(ColMap,RowMap);
ColX.Import(RowX,Importer,Insert);
if (NumDimensions > 1) ColY.Import(RowY,Importer,Insert);
if (NumDimensions > 2) ColZ.Import(RowZ,Importer,Insert);
// global row and column numbering
int* MyGlobalRowElements = RowMap.MyGlobalElements();
int* MyGlobalColElements = ColMap.MyGlobalElements();
// room for getrow()
int MaxNnz = RowMatrix_->MaxNumEntries();
std::vector<int> colInd(MaxNnz);
std::vector<double> colVal(MaxNnz);
std::vector<double> coord_i(3);
std::vector<double> coord_j(3);
// =================== //
// cycle over all rows //
// =================== //
for (int i = 0; i < NumMyRows ; i += NumPDEEqns_) {
int GlobalRow = MyGlobalRowElements[i];
if( i%NumPDEEqns_ == 0 ) { // do it just once for each block row
switch( NumDimensions ) {
case 3:
coord_i[2] = RowZ[i];
case 2:
coord_i[1] = RowY[i];
case 1:
coord_i[0] = RowX[i];
}
int NumEntries;
ML_CHK_ERR(RowMatrix_->ExtractMyRowCopy(i,MaxNnz,NumEntries,
&colVal[0],&colInd[0]));
// NOTE: for VBR matrices, the "real" value that will be used in
// the subsequent part of the code is only the one for the first
// equations. For each block, I replace values with the sum of
// the std::abs of each block entry.
for (int j = 0 ; j < NumEntries ; j += NumPDEEqns_) {
colVal[j] = std::fabs(colVal[j]);
for (int k = 1 ; k < NumPDEEqns_ ; ++k) {
colVal[j] += std::fabs(colVal[j+k]);
}
}
// work only on the first equations. Theta will blend the
// coordinate part with the sub of std::abs of row elements.
int GlobalCol;
double total = 0.0;
for (int j = 0 ; j < NumEntries ; j += NumPDEEqns_) {
if (colInd[j] >= NumMyRows)
continue;
if (colInd[j]%NumPDEEqns_ == 0) {
// insert diagonal later
if (colInd[j] != i) {
// get coordinates of this node
switch (NumDimensions) {
case 3:
coord_j[2] = ColZ[colInd[j]];
case 2:
coord_j[1] = ColY[colInd[j]];
case 1:
coord_j[0] = ColX[colInd[j]];
}
// d2 is the square of the distance between node `i' and
// node `j'
double d2 = (coord_i[0] - coord_j[0]) * (coord_i[0] - coord_j[0]) +
(coord_i[1] - coord_j[1]) * (coord_i[1] - coord_j[1]) +
(coord_i[2] - coord_j[2]) * (coord_i[2] - coord_j[2]);
if (d2 == 0.0) {
std::cerr << std::endl;
std::cerr << ErrorMsg_ << "distance between node " << i/NumPDEEqns_ << " and node "
<< colInd[j]/NumPDEEqns_ << std::endl
<< ErrorMsg_ << "is zero. Coordinates of these nodes are" << std::endl
<< ErrorMsg_ << "x_i = " << coord_i[0] << ", x_j = " << coord_j[0] << std::endl
<< ErrorMsg_ << "y_i = " << coord_i[1] << ", y_j = " << coord_j[1] << std::endl
<< ErrorMsg_ << "z_i = " << coord_i[2] << ", z_j = " << coord_j[2] << std::endl
<< ErrorMsg_ << "Now proceeding with distance = 1.0" << std::endl;
std::cerr << std::endl;
d2 = 1.0;
}
// blend d2 with the actual values of the matrix
// FIXME: am I useful?
double val = -(1.0 - theta) * (1.0 / d2) + theta * (colVal[j]);
GlobalCol = MyGlobalColElements[colInd[j]];
// insert this value on all rows
for (int k = 0 ; k < NumPDEEqns_ ; ++k) {
int row = GlobalRow + k;
int col = GlobalCol + k;
if (FakeMatrix->SumIntoGlobalValues(1,&row,1,&col,&val) != 0) {
ML_CHK_ERR(FakeMatrix->InsertGlobalValues(1,&row,1,&col,&val));
}
}
total -= val;
// put (j,i) element as well, only for in-process stuff.
// I have some problems with off-processor elements.
// It is here that I need the FE matrix.
if (SymmetricPattern == true && colInd[j] < NumMyRows ) {
for( int k=0 ; k<NumPDEEqns_ ; ++k ) {
int row = GlobalCol+k;
int col = GlobalRow+k;
if( FakeMatrix->SumIntoGlobalValues(1,&row,1,&col,&val) != 0 ) {
ML_CHK_ERR(FakeMatrix->InsertGlobalValues(1,&row,1,&col,&val));
}
}
total -= val;
}
}
}
}
// create lines with zero-row sum
for (int k = 0 ; k < NumPDEEqns_ ; ++k) {
int row = GlobalRow + k;
if (FakeMatrix->SumIntoGlobalValues(1,&row,1,&row,&total) != 0) {
if (FakeMatrix->InsertGlobalValues(1,&row,1,&row,&total) != 0)
ML_CHK_ERR(-9); // something went wrong
}
}
}
}
if (FakeMatrix->FillComplete())
ML_CHK_ERR(-5); // something went wrong
// stick pointer in Amat for level 0 (finest level)
ml_->Amat[LevelID_[0]].data = (void *)FakeMatrix;
// tell ML to keep the tentative prolongator
ML_Aggregate_Set_Reuse(agg_);
// pray that no bugs will tease us
return(0);
}
// ============================================================================
int ML_Epetra::MultiLevelPreconditioner::
CreateAuxiliaryMatrixVbr(Epetra_VbrMatrix* &FakeMatrix)
{
// FakeMatrix has already been created before
if (FakeMatrix == 0)
ML_CHK_ERR(-1); // something went wrong
int NumDimensions = 0;
double* x_coord = List_.get("x-coordinates", (double *)0);
if (x_coord != 0) ++NumDimensions;
// at least x-coordinates must be not null
if( NumDimensions == 0 ) {
std::cerr << ErrorMsg_ << "Option `aggregation: use auxiliary matrix' == true" << std::endl
<< ErrorMsg_ << "requires x-, y-, or z-coordinates." << std::endl
<< ErrorMsg_ << "You must specify them using options" << std::endl
<< ErrorMsg_ << "`x-coordinates' (and equivalently for" << std::endl
<< ErrorMsg_ << "y- and z-)." << std::endl;
ML_CHK_ERR(-2); // wrong parameters
}
double* y_coord = List_.get("y-coordinates", (double *)0);
if (y_coord != 0) ++NumDimensions;
double* z_coord = List_.get("z-coordinates", (double *)0);
if (z_coord != 0) ++NumDimensions;
// small check to avoid strange behavior
if( z_coord != 0 && y_coord == 0 ) {
std::cerr << ErrorMsg_ << "Something wrong: `y-coordinates'" << std::endl
<< ErrorMsg_ << "is null, while `z-coordinates' is not null" << std::endl;
ML_CHK_ERR(-3); // something went wrong
}
// usual crap to clutter the output
if( verbose_ ) {
std::cout << std::endl;
std::cout << PrintMsg_ << "*** Using auxiliary matrix to create the aggregates" << std::endl
<< PrintMsg_ << "*** Number of dimensions = " << NumDimensions << std::endl
<< PrintMsg_ << "*** (the version for Epetra_VbrMatrix is currently used)" << std::endl;
std::cout << std::endl;
}
const Epetra_BlockMap& RowMap = FakeMatrix->RowMap();
const Epetra_BlockMap& ColMap = FakeMatrix->ColMap();
int NumMyRowElements = RowMap.NumMyElements();
int NumMyColElements = ColMap.NumMyElements();
int* MyGlobalRowElements = RowMap.MyGlobalElements();
int* MyGlobalColElements = ColMap.MyGlobalElements();
// use point map to exchange coordinates
Epetra_Map PointRowMap(-1,NumMyRowElements,MyGlobalRowElements,0,Comm());
Epetra_Map PointColMap(-1,NumMyColElements,MyGlobalColElements,0,Comm());
Epetra_Vector RowX(PointRowMap);
Epetra_Vector RowY(PointRowMap);
Epetra_Vector RowZ(PointRowMap);
for (int i = 0 ; i < NumMyRowElements ; ++i) {
RowX[i] = x_coord[i];
if (NumDimensions > 1) RowY[i] = y_coord[i];
if (NumDimensions > 2) RowZ[i] = z_coord[i];
}
// create vectors containing coordinates for columns
// (this is useful only if MIS/ParMETIS are used)
Epetra_Vector ColX(PointColMap);
Epetra_Vector ColY(PointColMap);
Epetra_Vector ColZ(PointColMap);
// get coordinates for non-local nodes (in column map)
Epetra_Import Importer(PointColMap,PointRowMap);
ColX.Import(RowX,Importer,Insert);
if (NumDimensions > 1) ColY.Import(RowY,Importer,Insert);
if (NumDimensions > 2) ColZ.Import(RowZ,Importer,Insert);
// room for getrow()
int MaxNnz = FakeMatrix->MaxNumEntries();
std::vector<int> colInd(MaxNnz);
std::vector<double> colVal(MaxNnz);
std::vector<double> coord_i(3);
std::vector<double> coord_j(3);
// change the entries of FakeMatrix so that it corresponds to a discrete
// Laplacian. Note: This is not exactly the same as in the Crs case.
FakeMatrix->PutScalar(0.0);
for (int LocalRow = 0; LocalRow < NumMyRowElements ; ++LocalRow) {
int RowDim, NumBlockEntries;
int* BlockIndices;
Epetra_SerialDenseMatrix** RowValues;
switch (NumDimensions) {
case 3:
coord_i[2] = RowZ[LocalRow];
case 2:
coord_i[1] = RowY[LocalRow];
case 1:
coord_i[0] = RowX[LocalRow];
}
FakeMatrix->ExtractMyBlockRowView(LocalRow,RowDim,NumBlockEntries,
BlockIndices,RowValues);
// accumulator for zero row-sum
double total = 0.0;
for (int j = 0 ; j < NumBlockEntries ; ++j) {
int LocalCol = BlockIndices[j];
// insert diagonal later
if (LocalCol != LocalRow) {
// get coordinates of this node
switch (NumDimensions) {
case 3:
coord_j[2] = ColZ[LocalCol];
case 2:
coord_j[1] = ColY[LocalCol];
case 1:
coord_j[0] = ColX[LocalCol];
}
// d2 is the square of the distance between node `i' and
// node `j'
double d2 = (coord_i[0] - coord_j[0]) * (coord_i[0] - coord_j[0]) +
(coord_i[1] - coord_j[1]) * (coord_i[1] - coord_j[1]) +
(coord_i[2] - coord_j[2]) * (coord_i[2] - coord_j[2]);
if (d2 == 0.0) {
std::cerr << std::endl;
std::cerr << ErrorMsg_ << "distance between node " << LocalRow << " and node "
<< LocalCol << std::endl
<< ErrorMsg_ << "is zero. Coordinates of these nodes are" << std::endl
<< ErrorMsg_ << "x_i = " << coord_i[0] << ", x_j = " << coord_j[0] << std::endl
<< ErrorMsg_ << "y_i = " << coord_i[1] << ", y_j = " << coord_j[1] << std::endl
<< ErrorMsg_ << "z_i = " << coord_i[2] << ", z_j = " << coord_j[2] << std::endl
<< ErrorMsg_ << "Now proceeding with distance = 1.0" << std::endl;
std::cerr << std::endl;
d2 = 1.0;
}
for (int k = 0 ; k < RowValues[j]->M() ; ++k) {
for (int h = 0 ; h < RowValues[j]->N() ; ++h) {
if (k == h)
(*RowValues[j])(k,h) = - 1.0 / d2;
}
}
total += 1.0 /d2;
}
}
// check that the diagonal block exists
bool ok = false;
int DiagonalBlock = 0;
for (int j = 0 ; j < NumBlockEntries ; ++j) {
if (BlockIndices[j] == LocalRow) {
DiagonalBlock = j;
ok = true;
break;
}
}
assert (ok == true);
for (int k = 0 ; k < RowValues[DiagonalBlock]->N() ; ++k) {
(*RowValues[DiagonalBlock])(k,k) = total;
}
}
// tell ML to keep the tentative prolongator
ML_Aggregate_Set_Reuse(agg_);
// pray that no bugs will tease us
return(0);
}
Zoltan_CrsGraph::NewTypeRef
Zoltan_CrsGraph::
operator()( OriginalTypeRef orig )
{
origObj_ = &orig;
int err;
//Setup Load Balance Object
float version;
char * dummy = 0;
Zoltan::LoadBalance LB( 0, &dummy, &version );
err = LB.Create( dynamic_cast<const Epetra_MpiComm&>(orig.Comm()).Comm() );
if( err == ZOLTAN_OK ) err = LB.Set_Param( "LB_METHOD", "GRAPH" );
#ifdef HAVE_LIBPARMETIS
if( err == ZOLTAN_OK ) err = LB.Set_Param( "GRAPH_PACKAGE", "PARMETIS" );
if( err == ZOLTAN_OK ) err = LB.Set_Param( "PARMETIS_METHOD", partitionMethod_ );
#endif
//Setup Query Object
CrsGraph_Transpose transposeTransform;
Epetra_CrsGraph & TransGraph = transposeTransform( orig );
ZoltanQuery Query( orig, &TransGraph );
if( err == ZOLTAN_OK ) err = LB.Set_QueryObject( &Query );
if( err != ZOLTAN_OK )
{ cout << "Setup of Zoltan Load Balancing Objects FAILED!\n"; exit(0); }
//Generate Load Balance
int changes;
int num_gid_entries, num_lid_entries;
int num_import;
ZOLTAN_ID_PTR import_global_ids, import_local_ids;
int * import_procs;
int num_export;
ZOLTAN_ID_PTR export_global_ids, export_local_ids;
int * export_procs;
orig.Comm().Barrier();
err = LB.Balance( &changes,
&num_gid_entries, &num_lid_entries,
&num_import, &import_global_ids, &import_local_ids, &import_procs,
&num_export, &export_global_ids, &export_local_ids, &export_procs );
LB.Evaluate( 1, 0, 0, 0, 0, 0, 0 );
orig.Comm().Barrier();
//Generate New Element List
int numMyElements = orig.RowMap().NumMyElements();
vector<int> elementList( numMyElements );
orig.RowMap().MyGlobalElements( &elementList[0] );
int newNumMyElements = numMyElements - num_export + num_import;
vector<int> newElementList( newNumMyElements );
set<int> gidSet;
for( int i = 0; i < num_export; ++i )
gidSet.insert( export_global_ids[i] );
//Add unmoved indices to new list
int loc = 0;
for( int i = 0; i < numMyElements; ++i )
if( !gidSet.count( elementList[i] ) )
newElementList[loc++] = elementList[i];
//Add imports to end of list
for( int i = 0; i < num_import; ++i )
newElementList[loc+i] = import_global_ids[i];
//Free Zoltan Data
if( err == ZOLTAN_OK )
err = LB.Free_Data( &import_global_ids, &import_local_ids, &import_procs,
&export_global_ids, &export_local_ids, &export_procs );
//Create Import Map
NewRowMap_ = new Epetra_Map( orig.RowMap().NumGlobalElements(),
newNumMyElements,
&newElementList[0],
orig.RowMap().IndexBase(),
orig.RowMap().Comm() );
//Create Importer
Epetra_Import Importer( *NewRowMap_, orig.RowMap() );
//Create New Graph
Epetra_CrsGraph * NewGraph = new Epetra_CrsGraph( Copy, *NewRowMap_, 0 );
NewGraph->Import( orig, Importer, Insert );
NewGraph->FillComplete();
Zoltan::LoadBalance LB2( 0, &dummy, &version );
err = LB2.Create( dynamic_cast<const Epetra_MpiComm&>(orig.Comm()).Comm() );
if( err == ZOLTAN_OK ) err = LB2.Set_Param( "LB_METHOD", "GRAPH" );
#ifdef HAVE_LIBPARMETIS
if( err == ZOLTAN_OK ) err = LB2.Set_Param( "GRAPH_PACKAGE", "PARMETIS" );
if( err == ZOLTAN_OK ) err = LB2.Set_Param( "PARMETIS_METHOD", partitionMethod_ );
#endif
CrsGraph_Transpose transTrans;
Epetra_CrsGraph & trans2 = transTrans( *NewGraph );
ZoltanQuery query( *NewGraph, &trans2 );
if( err == ZOLTAN_OK ) err = LB2.Set_QueryObject( &query );
//err = LB2.Balance( &changes,
// &num_gid_entries, &num_lid_entries,
// &num_import, &import_global_ids, &import_local_ids, &import_procs,
// &num_export, &export_global_ids, &export_local_ids, &export_procs );
LB2.Evaluate( 1, 0, 0, 0, 0, 0, 0 );
newObj_ = NewGraph;
return *NewGraph;
}
int main(int argc, char *argv[])
{
int ierr = 0, i, j, forierr = 0;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm Comm;
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
//char tmp;
//if (Comm.MyPID()==0) cout << "Press any key to continue..."<< endl;
//if (Comm.MyPID()==0) cin >> tmp;
//Comm.Barrier();
Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
int MyPID = Comm.MyPID();
int NumProc = Comm.NumProc();
if (verbose && MyPID==0)
cout << Epetra_Version() << endl << endl;
if (verbose) cout << "Processor "<<MyPID<<" of "<< NumProc
<< " is alive."<<endl;
// Redefine verbose to only print on PE 0
if (verbose && Comm.MyPID()!=0) verbose = false;
int NumMyEquations = 20;
long long NumGlobalEquations = NumMyEquations*NumProc+EPETRA_MIN(NumProc,3);
if (MyPID < 3) NumMyEquations++;
// Construct a Source Map that puts approximately the same Number of equations on each processor in
// uniform global ordering
Epetra_Map SourceMap(NumGlobalEquations, NumMyEquations, 0LL, Comm);
// Get update list and number of local equations from newly created Map
int NumMyElements = SourceMap.NumMyElements();
long long * SourceMyGlobalElements = new long long[NumMyElements];
SourceMap.MyGlobalElements(SourceMyGlobalElements);
// Construct a Target Map that will contain:
// some unchanged elements (relative to the soure map),
// some permuted elements
// some off-processor elements
Epetra_Vector RandVec(SourceMap);
RandVec.Random(); // This creates a vector of random numbers between negative one and one.
long long *TargetMyGlobalElements = new long long[NumMyElements];
long long MinGID = SourceMap.MinMyGID64();
for (i=0; i< NumMyEquations/2; i++) TargetMyGlobalElements[i] = i + MinGID; // Half will be the same...
for (i=NumMyEquations/2; i<NumMyEquations; i++) {
int index = abs((int)(((double) (NumGlobalEquations-1) ) * RandVec[i]));
TargetMyGlobalElements[i] = EPETRA_MIN(NumGlobalEquations-1,(long long) EPETRA_MAX(0,index));
}
int NumSameIDs = 0;
int NumPermutedIDs = 0;
int NumRemoteIDs = 0;
bool StillContiguous = true;
for (i=0; i < NumMyEquations; i++) {
if (SourceMyGlobalElements[i]==TargetMyGlobalElements[i] && StillContiguous)
NumSameIDs++;
else if (SourceMap.MyGID(TargetMyGlobalElements[i])) {
StillContiguous = false;
NumPermutedIDs++;
}
else {
StillContiguous = false;
NumRemoteIDs++;
}
}
EPETRA_TEST_ERR(!(NumMyEquations==NumSameIDs+NumPermutedIDs+NumRemoteIDs),ierr);
Epetra_Map TargetMap((long long) -1, NumMyElements, TargetMyGlobalElements, 0LL, Comm);
// Create a multivector whose elements are GlobalID * (column number +1)
int NumVectors = 3;
Epetra_MultiVector SourceMultiVector(SourceMap, NumVectors);
for (j=0; j < NumVectors; j++)
for (i=0; i < NumMyElements; i++)
SourceMultiVector[j][i] = (double) SourceMyGlobalElements[i]*(j+1);
// Create a target multivector that we will fill using an Import
Epetra_MultiVector TargetMultiVector(TargetMap, NumVectors);
Epetra_Import Importer(TargetMap, SourceMap);
EPETRA_TEST_ERR(!(TargetMultiVector.Import(SourceMultiVector, Importer, Insert)==0),ierr);
// Test Target against expected values
forierr = 0;
for (j=0; j < NumVectors; j++)
for (i=0; i < NumMyElements; i++) {
if (TargetMultiVector[j][i]!= (double) TargetMyGlobalElements[i]*(j+1))
cout << "TargetMultiVector["<<i<<"]["<<j<<"] = " << TargetMultiVector[j][i]
<< " TargetMyGlobalElements[i]*(j+1) = " << TargetMyGlobalElements[i]*(j+1) << endl;
forierr += !(TargetMultiVector[j][i]== (double) TargetMyGlobalElements[i]*(j+1));
}
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "MultiVector Import using Importer Check OK" << endl << endl;
//////////////////////////////////////////////////////////////////////////////
// Now use Importer to do an export
Epetra_Vector TargetVector(SourceMap);
Epetra_Vector ExpectedTarget(SourceMap);
Epetra_Vector SourceVector(TargetMap);
NumSameIDs = Importer.NumSameIDs();
int NumPermuteIDs = Importer.NumPermuteIDs();
int NumExportIDs = Importer.NumExportIDs();
int *PermuteFromLIDs = Importer.PermuteFromLIDs();
int *ExportLIDs = Importer.ExportLIDs();
int *ExportPIDs = Importer.ExportPIDs();
for (i=0; i < NumSameIDs; i++) ExpectedTarget[i] = (double) (MyPID+1);
for (i=0; i < NumPermuteIDs; i++) ExpectedTarget[PermuteFromLIDs[i]] =
(double) (MyPID+1);
for (i=0; i < NumExportIDs; i++) ExpectedTarget[ExportLIDs[i]] +=
(double) (ExportPIDs[i]+1);
for (i=0; i < NumMyElements; i++) SourceVector[i] = (double) (MyPID+1);
EPETRA_TEST_ERR(!(TargetVector.Export(SourceVector, Importer, Add)==0),ierr);
forierr = 0;
for (i=0; i < NumMyElements; i++) {
if (TargetVector[i]!= ExpectedTarget[i])
cout << " TargetVector["<<i<<"] = " << TargetVector[i]
<< " ExpectedTarget["<<i<<"] = " << ExpectedTarget[i] << " on PE " << MyPID << endl;
forierr += !(TargetVector[i]== ExpectedTarget[i]);
}
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "Vector Export using Importer Check OK" << endl << endl;
//////////////////////////////////////////////////////////////////////////////
// Now use Importer to create a reverse exporter
TargetVector.PutScalar(0.0);
Epetra_Export ReversedImport(Importer);
EPETRA_TEST_ERR(!(TargetVector.Export(SourceVector, ReversedImport, Add)==0),ierr);
forierr = 0;
for (i=0; i < NumMyElements; i++) {
if (TargetVector[i]!= ExpectedTarget[i])
cout << " TargetVector["<<i<<"] = " << TargetVector[i]
<< " ExpectedTarget["<<i<<"] = " << ExpectedTarget[i] << " on PE " << MyPID << endl;
forierr += !(TargetVector[i]== ExpectedTarget[i]);
}
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "Vector Export using Reversed Importer Check OK" << endl << endl;
//////////////////////////////////////////////////////////////////////////////
// Now use Exporter to create a reverse importer
TargetVector.PutScalar(0.0);
Epetra_Import ReversedExport(ReversedImport);
EPETRA_TEST_ERR(!(TargetVector.Export(SourceVector, ReversedExport, Add)==0),ierr);
forierr = 0;
for (i=0; i < NumMyElements; i++) {
if (TargetVector[i]!= ExpectedTarget[i])
cout << " TargetVector["<<i<<"] = " << TargetVector[i]
<< " ExpectedTarget["<<i<<"] = " << ExpectedTarget[i] << " on PE " << MyPID << endl;
forierr += !(TargetVector[i]== ExpectedTarget[i]);
}
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "Vector Export using Reversed Exporter Check OK" << endl << endl;
//////////////////////////////////////////////////////////////////////////////////////////
// Build a tridiagonal system two ways:
// 1) From "standard" matrix view where equations are uniquely owned.
// 2) From 1D PDE view where nodes (equations) between processors are shared and partial contributions are done
// in parallel, then merged together at the end of the construction process.
//
//////////////////////////////////////////////////////////////////////////////////////////
// Construct a Standard Map that puts approximately the same number of equations on each processor in
// uniform global ordering
Epetra_Map StandardMap(NumGlobalEquations, NumMyEquations, 0LL, Comm);
// Get update list and number of local equations from newly created Map
NumMyElements = StandardMap.NumMyElements();
long long * StandardMyGlobalElements = new long long[NumMyElements];
StandardMap.MyGlobalElements(StandardMyGlobalElements);
// Create a standard Epetra_CrsGraph
Epetra_CrsGraph StandardGraph(Copy, StandardMap, 3);
EPETRA_TEST_ERR(StandardGraph.IndicesAreGlobal(),ierr);
EPETRA_TEST_ERR(StandardGraph.IndicesAreLocal(),ierr);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
long long *Indices = new long long[2];
int NumEntries;
forierr = 0;
for (i=0; i<NumMyEquations; i++)
{
if (StandardMyGlobalElements[i]==0)
{
Indices[0] = 1;
NumEntries = 1;
}
else if (StandardMyGlobalElements[i] == NumGlobalEquations-1)
{
Indices[0] = NumGlobalEquations-2;
NumEntries = 1;
}
else
{
Indices[0] = StandardMyGlobalElements[i]-1;
Indices[1] = StandardMyGlobalElements[i]+1;
NumEntries = 2;
}
forierr += !(StandardGraph.InsertGlobalIndices(StandardMyGlobalElements[i], NumEntries, Indices)==0);
forierr += !(StandardGraph.InsertGlobalIndices(StandardMyGlobalElements[i], 1, StandardMyGlobalElements+i)==0); // Put in the diagonal entry
}
EPETRA_TEST_ERR(forierr,ierr);
// Finish up
EPETRA_TEST_ERR(!(StandardGraph.IndicesAreGlobal()),ierr);
EPETRA_TEST_ERR(!(StandardGraph.FillComplete()==0),ierr);
EPETRA_TEST_ERR(!(StandardGraph.IndicesAreLocal()),ierr);
EPETRA_TEST_ERR(StandardGraph.StorageOptimized(),ierr);
StandardGraph.OptimizeStorage();
EPETRA_TEST_ERR(!(StandardGraph.StorageOptimized()),ierr);
EPETRA_TEST_ERR(StandardGraph.UpperTriangular(),ierr);
EPETRA_TEST_ERR(StandardGraph.LowerTriangular(),ierr);
// Create Epetra_CrsMatrix using the just-built graph
Epetra_CrsMatrix StandardMatrix(Copy, StandardGraph);
EPETRA_TEST_ERR(StandardMatrix.IndicesAreGlobal(),ierr);
EPETRA_TEST_ERR(!(StandardMatrix.IndicesAreLocal()),ierr);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values = new double[2];
Values[0] = -1.0; Values[1] = -1.0;
double two = 2.0;
forierr = 0;
for (i=0; i<NumMyEquations; i++)
{
if (StandardMyGlobalElements[i]==0)
{
Indices[0] = 1;
NumEntries = 1;
}
else if (StandardMyGlobalElements[i] == NumGlobalEquations-1)
{
Indices[0] = NumGlobalEquations-2;
NumEntries = 1;
}
else
{
Indices[0] = StandardMyGlobalElements[i]-1;
Indices[1] = StandardMyGlobalElements[i]+1;
NumEntries = 2;
}
forierr += !(StandardMatrix.ReplaceGlobalValues(StandardMyGlobalElements[i], NumEntries, Values, Indices)==0);
// Put in the diagonal entry
forierr += !(StandardMatrix.ReplaceGlobalValues(StandardMyGlobalElements[i], 1, &two, StandardMyGlobalElements+i)==0);
}
EPETRA_TEST_ERR(forierr,ierr);
// Finish up
EPETRA_TEST_ERR(!(StandardMatrix.IndicesAreLocal()),ierr);
EPETRA_TEST_ERR(!(StandardMatrix.FillComplete()==0),ierr);
EPETRA_TEST_ERR(!(StandardMatrix.IndicesAreLocal()),ierr);
// EPETRA_TEST_ERR((StandardMatrix.StorageOptimized()),ierr);
EPETRA_TEST_ERR((StandardMatrix.OptimizeStorage()),ierr);
EPETRA_TEST_ERR(!(StandardMatrix.StorageOptimized()),ierr);
EPETRA_TEST_ERR(StandardMatrix.UpperTriangular(),ierr);
EPETRA_TEST_ERR(StandardMatrix.LowerTriangular(),ierr);
// Construct an Overlapped Map of StandardMap that include the endpoints from two neighboring processors.
int OverlapNumMyElements;
long long OverlapMinMyGID;
OverlapNumMyElements = NumMyElements + 1;
if (MyPID==0) OverlapNumMyElements--;
if (MyPID==0) OverlapMinMyGID = StandardMap.MinMyGID64();
else OverlapMinMyGID = StandardMap.MinMyGID64()-1;
long long * OverlapMyGlobalElements = new long long[OverlapNumMyElements];
for (i=0; i< OverlapNumMyElements; i++) OverlapMyGlobalElements[i] = OverlapMinMyGID + i;
Epetra_Map OverlapMap((long long) -1, OverlapNumMyElements, OverlapMyGlobalElements, 0LL, Comm);
// Create the Overlap Epetra_Matrix
Epetra_CrsMatrix OverlapMatrix(Copy, OverlapMap, 4);
EPETRA_TEST_ERR(OverlapMatrix.IndicesAreGlobal(),ierr);
EPETRA_TEST_ERR(OverlapMatrix.IndicesAreLocal(),ierr);
// Add matrix element one cell at a time.
// Each cell does an incoming and outgoing flux calculation
double pos_one = 1.0;
double neg_one = -1.0;
forierr = 0;
for (i=0; i<OverlapNumMyElements; i++)
{
long long node_left = OverlapMyGlobalElements[i]-1;
long long node_center = node_left + 1;
long long node_right = node_left + 2;
if (i>0) {
if (node_left>-1)
forierr += !(OverlapMatrix.InsertGlobalValues(node_center, 1, &neg_one, &node_left)==0);
forierr += !(OverlapMatrix.InsertGlobalValues(node_center, 1, &pos_one, &node_center)==0);
}
if (i<OverlapNumMyElements-1) {
forierr += !(OverlapMatrix.InsertGlobalValues(node_center, 1, &pos_one, &node_center)==0);
if (node_right<NumGlobalEquations)
forierr += !(OverlapMatrix.InsertGlobalValues(node_center, 1, &neg_one, &node_right)==0);
}
}
EPETRA_TEST_ERR(forierr,ierr);
// Handle endpoints
if (MyPID==0) {
long long node_center = 0;
EPETRA_TEST_ERR(!(OverlapMatrix.InsertGlobalValues(node_center, 1, &pos_one, &node_center)==0),ierr);
}
if (MyPID==NumProc-1) {
long long node_center = OverlapMyGlobalElements[OverlapNumMyElements-1];
EPETRA_TEST_ERR(!(OverlapMatrix.InsertGlobalValues(node_center, 1, &pos_one, &node_center)==0),ierr);
}
EPETRA_TEST_ERR(!(OverlapMatrix.FillComplete()==0),ierr);
// Make a gathered matrix from OverlapMatrix. It should be identical to StandardMatrix
Epetra_CrsMatrix GatheredMatrix(Copy, StandardGraph);
Epetra_Export Exporter(OverlapMap, StandardMap);
EPETRA_TEST_ERR(!(GatheredMatrix.Export(OverlapMatrix, Exporter, Add)==0),ierr);
EPETRA_TEST_ERR(!(GatheredMatrix.FillComplete()==0),ierr);
// Check if entries of StandardMatrix and GatheredMatrix are identical
int StandardNumEntries, GatheredNumEntries;
int * StandardIndices, * GatheredIndices;
double * StandardValues, * GatheredValues;
int StandardNumMyNonzeros = StandardMatrix.NumMyNonzeros();
int GatheredNumMyNonzeros = GatheredMatrix.NumMyNonzeros();
EPETRA_TEST_ERR(!(StandardNumMyNonzeros==GatheredNumMyNonzeros),ierr);
int StandardNumMyRows = StandardMatrix.NumMyRows();
int GatheredNumMyRows = GatheredMatrix.NumMyRows();
EPETRA_TEST_ERR(!(StandardNumMyRows==GatheredNumMyRows),ierr);
forierr = 0;
for (i=0; i< StandardNumMyRows; i++)
{
forierr += !(StandardMatrix.ExtractMyRowView(i, StandardNumEntries, StandardValues, StandardIndices)==0);
forierr += !(GatheredMatrix.ExtractMyRowView(i, GatheredNumEntries, GatheredValues, GatheredIndices)==0);
forierr += !(StandardNumEntries==GatheredNumEntries);
for (j=0; j < StandardNumEntries; j++) {
//if (StandardIndices[j]!=GatheredIndices[j])
// cout << "MyPID = " << MyPID << " i = " << i << " StandardIndices[" << j << "] = " << StandardIndices[j]
// << " GatheredIndices[" << j << "] = " << GatheredIndices[j] << endl;
//if (StandardValues[j]!=GatheredValues[j])
//cout << "MyPID = " << MyPID << " i = " << i << " StandardValues[" << j << "] = " << StandardValues[j]
// << " GatheredValues[" << j << "] = " << GatheredValues[j] << endl;
forierr += !(StandardIndices[j]==GatheredIndices[j]);
forierr += !(StandardValues[j]==GatheredValues[j]);
}
}
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "Matrix Export Check OK" << endl << endl;
//Do Again with use of Epetra_OffsetIndex object for speed
Epetra_OffsetIndex OffsetIndex( OverlapMatrix.Graph(), GatheredMatrix.Graph(), Exporter );
EPETRA_TEST_ERR(!(GatheredMatrix.Export(OverlapMatrix, Exporter, Add)==0),ierr);
if (verbose) cout << "Optimized Matrix Export Check OK" << endl << endl;
bool passed;
Epetra_LongLongVector v1(StandardMap); v1.PutValue(2);
Epetra_LongLongVector v2(StandardMap); v2.PutValue(3);
Epetra_Export identExporter(StandardMap,StandardMap); // Identity exporter
EPETRA_TEST_ERR(!(v2.Export(v1, identExporter, Insert)==0),ierr);
passed = (v2.MinValue()==2);
EPETRA_TEST_ERR(!passed,ierr);
v1.PutValue(1);
Epetra_Import identImporter(StandardMap,StandardMap); // Identity importer
EPETRA_TEST_ERR(!(v2.Import(v1, identExporter, Insert)==0),ierr);
passed = passed && (v2.MaxValue()==1);
EPETRA_TEST_ERR(!passed,ierr);
if (verbose) {
if (passed) cout << "Identity Import/Export Check OK" << endl << endl;
else cout << "Identity Import/Export Check Failed" << endl << endl;
}
int NumSubMapElements = StandardMap.NumMyElements()/2;
int SubStart = Comm.MyPID();
NumSubMapElements = EPETRA_MIN(NumSubMapElements,StandardMap.NumMyElements()-SubStart);
Epetra_Map SubMap((long long) -1, NumSubMapElements, StandardMyGlobalElements+SubStart, 0LL, Comm);
Epetra_LongLongVector v3(View, SubMap, SubMap.MyGlobalElements64()); // Fill v3 with GID values for variety
Epetra_Export subExporter(SubMap, StandardMap); // Export to a subset of indices of standard map
EPETRA_TEST_ERR(!(v2.Export(v3,subExporter,Insert)==0),ierr);
forierr = 0;
for (i=0; i<SubMap.NumMyElements(); i++) {
int i1 = StandardMap.LID(SubMap.GID64(i));
forierr += !(v3[i]==v2[i1]);
}
EPETRA_TEST_ERR(forierr,ierr);
Epetra_Import subImporter(StandardMap, SubMap); // Import to a subset of indices of standard map
EPETRA_TEST_ERR(!(v1.Import(v3,subImporter,Insert)==0),ierr);
for (i=0; i<SubMap.NumMyElements(); i++) {
int i1 = StandardMap.LID(SubMap.GID64(i));
forierr += !(v3[i]==v1[i1]);
}
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) {
if (forierr==0) cout << "SubMap Import/Export Check OK" << endl << endl;
else cout << "SubMap Import/Export Check Failed" << endl << endl;
}
#ifdef DOESNT_WORK_IN_PARALLEL
forierr = special_submap_import_test(Comm);
EPETRA_TEST_ERR(forierr, ierr);
if (verbose) {
if (forierr==0) cout << "Special SubMap Import Check OK" << endl << endl;
else cout << "Special SubMap Import Check Failed" << endl << endl;
}
#endif
forierr = alternate_import_constructor_test(Comm);
EPETRA_TEST_ERR(forierr, ierr);
if (verbose) {
if (forierr==0) cout << "Alternative Import Constructor Check OK" << endl << endl;
else cout << "Alternative Import Constructor Check Failed" << endl << endl;
}
// Release all objects
delete [] SourceMyGlobalElements;
delete [] TargetMyGlobalElements;
delete [] OverlapMyGlobalElements;
delete [] StandardMyGlobalElements;
delete [] Values;
delete [] Indices;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
/* end main
*/
return ierr ;
}
//=============================================================================
int Amesos_Dscpack::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
// MS // some checks on matrix size
if (RowMatrixA->NumGlobalRows() != RowMatrixA->NumGlobalCols())
AMESOS_CHK_ERR(-1);
// Convert vector b to a vector in the form that DSCPACK needs it
//
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong with input
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-2);
double *dscmapXvalues ;
int dscmapXlda ;
Epetra_MultiVector dscmapX(DscRowMap(),NumVectors) ;
int ierr;
AMESOS_CHK_ERR(dscmapX.ExtractView(&dscmapXvalues,&dscmapXlda));
assert (dscmapXlda == NumLocalCols);
double *dscmapBvalues ;
int dscmapBlda ;
Epetra_MultiVector dscmapB(DscRowMap(), NumVectors ) ;
ierr = dscmapB.ExtractView( &dscmapBvalues, &dscmapBlda );
AMESOS_CHK_ERR(ierr);
assert( dscmapBlda == NumLocalCols ) ;
AMESOS_CHK_ERR(dscmapB.Import(*vecB, Importer(), Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
ResetTimer(0);
// MS // now solve the problem
std::vector<double> ValuesInNewOrder( NumLocalCols ) ;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
for ( int j =0 ; j < NumVectors; j++ ) {
for ( int i = 0; i < NumLocalCols; i++ ) {
ValuesInNewOrder[i] = dscmapBvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapBlda ] ;
}
AMESOS_CHK_ERR( DSC_InputRhsLocalVec ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
AMESOS_CHK_ERR( DSC_Solve ( PrivateDscpackData_->MyDSCObject_ ) ) ;
AMESOS_CHK_ERR( DSC_GetLocalSolution ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
for ( int i = 0; i < NumLocalCols; i++ ) {
dscmapXvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapXlda ] = ValuesInNewOrder[i];
}
}
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
ResetTimer(0);
ResetTimer(1);
vecX->Export( dscmapX, Importer(), Insert ) ;
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
ComputeTrueResidual(*(GetProblem()->GetMatrix()), *vecX, *vecB,
false, "Amesos_Dscpack");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Dscpack");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
NumSolve_++;
return(0) ;
}
// main driver
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
if (Comm.NumProc() != 2) {
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(0);
}
int NumMyElements = 0; // NODES assigned to this processor
int NumMyExternalElements = 0; // nodes used by this proc, but not hosted
int NumMyTotalElements = 0;
int FE_NumMyElements = 0; // TRIANGLES assigned to this processor
int * MyGlobalElements = 0; // nodes assigned to this processor
Epetra_IntSerialDenseMatrix T; // store the grid connectivity
int MyPID=Comm.MyPID();
cout << MyPID << endl;
switch( MyPID ) {
case 0:
NumMyElements = 3;
NumMyExternalElements = 2;
NumMyTotalElements = NumMyElements + NumMyExternalElements;
FE_NumMyElements = 3;
MyGlobalElements = new int[NumMyTotalElements];
MyGlobalElements[0] = 0;
MyGlobalElements[1] = 4;
MyGlobalElements[2] = 3;
MyGlobalElements[3] = 1;
MyGlobalElements[4] = 5;
break;
case 1:
NumMyElements = 3;
NumMyExternalElements = 2;
NumMyTotalElements = NumMyElements + NumMyExternalElements;
FE_NumMyElements = 3;
MyGlobalElements = new int[NumMyTotalElements];
MyGlobalElements[0] = 1;
MyGlobalElements[1] = 2;
MyGlobalElements[2] = 5;
MyGlobalElements[3] = 0;
MyGlobalElements[4] = 4;
break;
}
// build Map corresponding to update
Epetra_Map Map(-1,NumMyElements,MyGlobalElements,0,Comm);
// vector containing coordinates BEFORE exchanging external nodes
Epetra_Vector CoordX_noExt(Map);
Epetra_Vector CoordY_noExt(Map);
switch( MyPID ) {
case 0:
T.Shape(3,FE_NumMyElements);
// fill x-coordinates
CoordX_noExt[0] = 0.0;
CoordX_noExt[1] = 1.0;
CoordX_noExt[2] = 0.0;
// fill y-coordinates
CoordY_noExt[0] = 0.0;
CoordY_noExt[1] = 1.0;
CoordY_noExt[2] = 1.0;
// fill connectivity
T(0,0) = 0; T(0,1) = 4; T(0,2) = 3;
T(1,0) = 0; T(1,1) = 1; T(1,2) = 4;
T(2,0) = 4; T(2,1) = 1; T(2,2) = 5;
break;
case 1:
T.Shape(3,FE_NumMyElements);
// fill x-coordinates
CoordX_noExt[0] = 1.0;
CoordX_noExt[1] = 2.0;
CoordX_noExt[2] = 2.0;
// fill y-coordinates
CoordY_noExt[0] = 0.0;
CoordY_noExt[1] = 0.0;
CoordY_noExt[2] = 1.0;
// fill connectivity
T(0,0) = 0; T(0,1) = 1; T(0,2) = 4;
T(1,0) = 1; T(1,1) = 5; T(1,2) = 4;
T(2,0) = 1; T(2,1) = 2; T(2,2) = 5;
break;
}
// - - - - - - - - - - - - - - - - - - - - //
// E X T E R N A L N O D E S S E T U P //
// - - - - - - - - - - - - - - - - - - - - //
// build target map to exchange the valus of external nodes
Epetra_Map TargetMap(-1,NumMyTotalElements,
MyGlobalElements, 0, Comm);
// !@# rename Map -> SourceMap ?????
Epetra_Import Importer(TargetMap,Map);
Epetra_Vector CoordX(TargetMap);
Epetra_Vector CoordY(TargetMap);
CoordX.Import(CoordX_noExt,Importer,Insert);
CoordY.Import(CoordY_noExt,Importer,Insert);
// now CoordX_noExt and CoordY_noExt are no longer required
// NOTE: better to construct CoordX and CoordY as MultiVector
// - - - - - - - - - - - - //
// M A T R I X S E T U P //
// - - - - - - - - - - - - //
// build the CRS matrix corresponding to the grid
// some vectors are allocated
const int MaxNnzRow = 5;
Epetra_CrsMatrix A(Copy,Map,MaxNnzRow);
int Element, MyRow, GlobalRow, GlobalCol, i, j, k;
Epetra_IntSerialDenseMatrix Struct; // temp to create the matrix connectivity
Struct.Shape(NumMyElements,MaxNnzRow);
for( i=0 ; i<NumMyElements ; ++i )
for( j=0 ; j<MaxNnzRow ; ++j )
Struct(i,j) = -1;
// cycle over all the finite elements
for( Element=0 ; Element<FE_NumMyElements ; ++Element ) {
// cycle over each row
for( i=0 ; i<3 ; ++i ) {
// get the global and local number of this row
GlobalRow = T(Element,i);
MyRow = A.LRID(GlobalRow);
if( MyRow != -1 ) { // only rows stored on this proc
// cycle over the columns
for( j=0 ; j<3 ; ++j ) {
// get the global number only of this column
GlobalCol = T(Element,j);
// look if GlobalCol was already put in Struct
for( k=0 ; k<MaxNnzRow ; ++k ) {
if( Struct(MyRow,k) == GlobalCol ||
Struct(MyRow,k) == -1 ) break;
}
if( Struct(MyRow,k) == -1 ) { // new entry
Struct(MyRow,k) = GlobalCol;
} else if( Struct(MyRow,k) != GlobalCol ) {
// maybe not enough space has beenn allocated
cerr << "ERROR: not enough space for element "
<< GlobalRow << "," << GlobalCol << endl;
return( 0 );
}
}
}
}
}
int * Indices = new int [MaxNnzRow];
double * Values = new double [MaxNnzRow];
for( i=0 ; i<MaxNnzRow ; ++i ) Values[i] = 0.0;
// now use Struct to fill build the matrix structure
for( int Row=0 ; Row<NumMyElements ; ++Row ) {
int Length = 0;
for( int j=0 ; j<MaxNnzRow ; ++j ) {
if( Struct(Row,j) == -1 ) break;
Indices[Length] = Struct(Row,j);
Length++;
}
GlobalRow = MyGlobalElements[Row];
A.InsertGlobalValues(GlobalRow, Length, Values, Indices);
}
// replace global numbering with local one in T
for( int Element=0 ; Element<FE_NumMyElements ; ++Element ) {
for( int i=0 ; i<3 ; ++i ) {
int global = T(Element,i);
int local = find(MyGlobalElements,NumMyTotalElements,
global);
if( global == -1 ) {
cerr << "ERROR\n";
return( EXIT_FAILURE );
}
T(Element,i) = local;
}
}
// - - - - - - - - - - - - - - //
// M A T R I X F I L L - I N //
// - - - - - - - - - - - - - - //
// room for the local matrix
Epetra_SerialDenseMatrix Ke;
Ke.Shape(3,3);
// now fill the matrix
for( int Element=0 ; Element<FE_NumMyElements ; ++Element ) {
// variables used inside
int GlobalRow;
int MyRow;
int GlobalCol;
double x_triangle[3];
double y_triangle[3];
// get the spatial coordinate of each local node
for( int i=0 ; i<3 ; ++i ) {
MyRow = T(Element,i);
y_triangle[i] = CoordX[MyRow];
x_triangle[i] = CoordY[MyRow];
}
// compute the local matrix for Element
compute_loc_matrix( x_triangle, y_triangle,Ke );
// insert it in the global one
// cycle over each row
for( int i=0 ; i<3 ; ++i ) {
// get the global and local number of this row
MyRow = T(Element,i);
if( MyRow < NumMyElements ) {
for( int j=0 ; j<3 ; ++j ) {
// get global column number
GlobalRow = MyGlobalElements[MyRow];
GlobalCol = MyGlobalElements[T(Element,j)];
A.SumIntoGlobalValues(GlobalRow,1,&(Ke(i,j)),&GlobalCol);
}
}
}
}
A.FillComplete();
// - - - - - - - - - - - - - //
// R H S & S O L U T I O N //
// - - - - - - - - - - - - - //
Epetra_Vector x(Map), b(Map);
x.Random(); b.PutScalar(0.0);
// Solution can be obtained using Aztecoo
// free memory before leaving
delete MyGlobalElements;
delete Indices;
delete Values;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return( EXIT_SUCCESS );
} /* main */
