//----------------------------------------------------------------------------
Epetra_FECrsMatrix::Epetra_FECrsMatrix(Epetra_DataAccess CV,
const Epetra_FECrsGraph& graph,
bool ignoreNonLocalEntries)
: Epetra_CrsMatrix(CV, graph),
myFirstRow_(0),
myNumRows_(0),
ignoreNonLocalEntries_(ignoreNonLocalEntries),
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
nonlocalRows_int_(),
nonlocalCols_int_(),
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
nonlocalRows_LL_(),
nonlocalCols_LL_(),
#endif
nonlocalCoefs_(),
workData_(128),
useNonlocalMatrix_ (graph.UseNonlocalGraph() && graph.nonlocalGraph_ != 0),
nonlocalMatrix_ (useNonlocalMatrix_ ?
new Epetra_CrsMatrix(Copy,*graph.nonlocalGraph_) : NULL),
sourceMap_(NULL),
colMap_(NULL),
exporter_(NULL),
tempMat_(NULL)
{
myFirstRow_ = RowMap().MinMyGID64();
myNumRows_ = RowMap().NumMyElements();
}
//----------------------------------------------------------------------------
void Epetra_FECrsMatrix::DeleteMemory()
{
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
if (RowMap().GlobalIndicesInt()) {
nonlocalRows_int_.clear();
nonlocalCols_int_.clear();
}
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
if (RowMap().GlobalIndicesLongLong()) {
nonlocalRows_LL_.clear();
nonlocalCols_LL_.clear();
}
#endif
nonlocalCoefs_.clear();
if (nonlocalMatrix_ != 0)
delete nonlocalMatrix_;
if ( sourceMap_ )
delete sourceMap_;
if ( colMap_ )
delete colMap_;
if ( exporter_ )
delete exporter_;
if ( tempMat_ )
delete tempMat_;
}
int Epetra_FECrsMatrix::GlobalAssemble(const Epetra_Map& domain_map,
const Epetra_Map& range_map,
bool callFillComplete,
Epetra_CombineMode combineMode,
bool save_off_and_reuse_map_exporter)
{
if(!domain_map.GlobalIndicesTypeMatch(range_map))
throw ReportError("Epetra_FECrsMatrix::GlobalAssemble: cannot be called with different indices types for domainMap and rangeMap", -1);
if(!RowMap().GlobalIndicesTypeMatch(domain_map))
throw ReportError("Epetra_FECrsMatrix::GlobalAssemble: cannot be called with different indices types for row map and incoming rangeMap", -1);
if(RowMap().GlobalIndicesInt())
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
return GlobalAssemble<int>(domain_map, range_map, callFillComplete, combineMode, save_off_and_reuse_map_exporter);
#else
throw ReportError("Epetra_FECrsMatrix::GlobalAssemble: ERROR, GlobalIndicesInt but no API for it.",-1);
#endif
if(RowMap().GlobalIndicesLongLong())
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
return GlobalAssemble<long long>(domain_map, range_map, callFillComplete, combineMode, save_off_and_reuse_map_exporter);
#else
throw ReportError("Epetra_FECrsMatrix::GlobalAssemble: ERROR, GlobalIndicesLongLong but no API for it.",-1);
#endif
throw ReportError("Epetra_FECrsMatrix::GlobalAssemble: Internal error, unable to determine global index type of maps", -1);
}
int Epetra_FECrsMatrix::InputNonlocalValue(int rowoffset,
int_type col, double value,
int mode)
{
if(!RowMap().template GlobalIndicesIsType<int_type>())
throw ReportError("Epetra_FECrsMatrix::InputNonlocalValue mismatch between argument types (int/long long) and map type.", -1);
std::vector<int_type>& colIndices = nonlocalCols<int_type>()[rowoffset];
std::vector<double>& coefs = nonlocalCoefs_[rowoffset];
typename std::vector<int_type>::iterator it =
std::lower_bound(colIndices.begin(), colIndices.end(), col);
if (it == colIndices.end() || *it != col) {
int offset = (int) (it - colIndices.begin());
colIndices.insert(it, col);
std::vector<double>::iterator dit = coefs.begin()+offset;
coefs.insert(dit, value);
return 0;
}
int coloffset = (int) (it - colIndices.begin());
if (mode == SUMINTO || mode == INSERT) {
coefs[coloffset] += value;
}
else {
coefs[coloffset] = value;
}
return(0);
}
int Epetra_FECrsMatrix::InputGlobalValues(int numRows, const int_type* rows,
int numCols, const int_type* cols,
const double* values,
int format, int mode)
{
if(!RowMap().template GlobalIndicesIsType<int_type>())
throw ReportError("Epetra_FECrsMatrix::InputGlobalValues mismatch between argument types (int/long long) and map type.", -1);
if (format == Epetra_FECrsMatrix::ROW_MAJOR) {
return InputGlobalValues_RowMajor(numRows, rows, numCols, cols, values, mode);
}
workData_.resize(numCols);
int returncode = 0;
for(int i=0; i<numRows; ++i) {
//copy each row out of the column-major values array, so we can pass it
//to a row-major input function.
for(int j=0; j<numCols; ++j) {
workData_[j] = values[i+j*numRows];
}
int err = InputGlobalValues_RowMajor(1, &rows[i], numCols, cols, &workData_[0], mode);
if (err < 0) return err;
returncode += err;
}
return(returncode);
}
int Epetra_FECrsMatrix::SumIntoGlobalValues(long long GlobalRow, int NumEntries,
const double* values, const long long* Indices)
{
if(RowMap().GlobalIndicesLongLong())
return SumIntoGlobalValues<long long>(GlobalRow, NumEntries, values, Indices);
else
throw ReportError("Epetra_FECrsMatrix::SumIntoGlobalValues long long version called for a matrix that is not long long.", -1);
}
int Epetra_FECrsMatrix::SumIntoGlobalValues(int GlobalRow, int NumEntries,
const double* values, const int* Indices)
{
if(RowMap().GlobalIndicesInt())
return SumIntoGlobalValues<int>(GlobalRow, NumEntries, values, Indices);
else
throw ReportError("Epetra_FECrsMatrix::SumIntoGlobalValues int version called for a matrix that is not int.", -1);
}
int Epetra_FECrsMatrix::InputGlobalValues(int numRows, const int_type* rows,
int numCols, const int_type* cols,
const double*const* values,
int format, int mode)
{
if(!RowMap().template GlobalIndicesIsType<int_type>())
throw ReportError("Epetra_FECrsMatrix::InputGlobalValues mismatch between argument types (int/long long) and map type.", -1);
if (format != Epetra_FECrsMatrix::ROW_MAJOR &&
format != Epetra_FECrsMatrix::COLUMN_MAJOR) {
std::cerr << "Epetra_FECrsMatrix: unrecognized format specifier."<< std::endl;
return(-1);
}
if (format == Epetra_FECrsMatrix::COLUMN_MAJOR) {
workData_.resize(numCols);
}
int returncode = 0;
for(int i=0; i<numRows; ++i) {
if (format == Epetra_FECrsMatrix::ROW_MAJOR) {
returncode += InputGlobalValues_RowMajor(1, &rows[i], numCols, cols,
values[i], mode);
if (returncode < 0) return returncode;
continue;
}
//If we get to here, the data is in column-major order.
double* valuesptr = &workData_[0];
//Since the data is in column-major order, then we copy the i-th row
//of the values table into workData_, in order to have the row in
//contiguous memory.
//This is slow and not thread-safe.
for(int j=0; j<numCols; ++j) {
valuesptr[j] = values[j][i];
}
returncode += InputGlobalValues_RowMajor(1, &rows[i], numCols, cols, valuesptr, mode);
if (returncode < 0) return returncode;
}
return(returncode);
}
int Epetra_FECrsMatrix::InsertNonlocalRow(int_type row, typename std::vector<int_type>::iterator iter)
{
if(!RowMap().template GlobalIndicesIsType<int_type>())
throw ReportError("Epetra_FECrsMatrix::InsertNonlocalRow mismatch between argument types (int/long long) and map type.", -1);
std::vector<int_type>& nonlocalRows_var = nonlocalRows<int_type>();
std::vector<std::vector<int_type> >& nonlocalCols_var = nonlocalCols<int_type>();
int offset = (int) (iter - nonlocalRows_var.begin());
nonlocalRows_var.insert(iter, row);
typename std::vector<std::vector<int_type> >::iterator cols_iter = nonlocalCols_var.begin() + offset;
nonlocalCols_var.insert(cols_iter, std::vector<int_type>());
std::vector<std::vector<double> >::iterator coefs_iter = nonlocalCoefs_.begin() + offset;
nonlocalCoefs_.insert(coefs_iter, std::vector<double>());
return(0);
}
//=============================================================================
int Epetra_FastCrsMatrix::Multiply(bool TransA, const Epetra_Vector& x, Epetra_Vector& y) const {
//
// This function forms the product y = A * x or y = A' * x
//
int i, j;
double * xp = (double*)x.Values();
double *yp = (double*)y.Values();
int NumMyCols_ = NumMyCols();
if (!TransA) {
// If we have a non-trivial importer, we must import elements that are permuted or are on other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=1) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),1); // Create import vector if needed
ImportVector_->Import(x, *Importer(), Insert);
xp = (double*)ImportVector_->Values();
}
// If we have a non-trivial exporter, we must export elements that are permuted or belong to other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=1) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),1); // Create Export vector if needed
yp = (double*)ExportVector_->Values();
}
// Do actual computation
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
double sum = 0.0;
for (j=0; j < NumEntries; j++) sum += RowValues[j] * xp[RowIndices[j]];
yp[i] = sum;
}
if (Exporter()!=0) y.Export(*ExportVector_, *Exporter(), Add); // Fill y with Values from export vector
}
else { // Transpose operation
// If we have a non-trivial exporter, we must import elements that are permuted or are on other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=1) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),1); // Create Export vector if needed
ExportVector_->Import(x, *Exporter(), Insert);
xp = (double*)ExportVector_->Values();
}
// If we have a non-trivial importer, we must export elements that are permuted or belong to other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=1) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),1); // Create import vector if needed
yp = (double*)ImportVector_->Values();
}
// Do actual computation
for (i=0; i < NumMyCols_; i++) yp[i] = 0.0; // Initialize y for transpose multiply
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (j=0; j < NumEntries; j++) yp[RowIndices[j]] += RowValues[j] * xp[i];
}
if (Importer()!=0) y.Export(*ImportVector_, *Importer(), Add); // Fill y with Values from export vector
}
UpdateFlops(2*NumGlobalNonzeros64());
return(0);
}
//=======================================================
EpetraExt_HypreIJMatrix::EpetraExt_HypreIJMatrix(HYPRE_IJMatrix matrix)
: Epetra_BasicRowMatrix(Epetra_MpiComm(hypre_IJMatrixComm(matrix))),
Matrix_(matrix),
ParMatrix_(0),
NumMyRows_(-1),
NumGlobalRows_(-1),
NumGlobalCols_(-1),
MyRowStart_(-1),
MyRowEnd_(-1),
MatType_(-1),
TransposeSolve_(false),
SolveOrPrec_(Solver)
{
IsSolverSetup_ = new bool[1];
IsPrecondSetup_ = new bool[1];
IsSolverSetup_[0] = false;
IsPrecondSetup_[0] = false;
// Initialize default values for global variables
int ierr = 0;
ierr += InitializeDefaults();
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't initialize default values.");
// Create array of global row ids
Teuchos::Array<int> GlobalRowIDs; GlobalRowIDs.resize(NumMyRows_);
for (int i = MyRowStart_; i <= MyRowEnd_; i++) {
GlobalRowIDs[i-MyRowStart_] = i;
}
// Create array of global column ids
int new_value = 0; int entries = 0;
std::set<int> Columns;
int num_entries;
double *values;
int *indices;
for(int i = 0; i < NumMyRows_; i++){
ierr += HYPRE_ParCSRMatrixGetRow(ParMatrix_, i+MyRowStart_, &num_entries, &indices, &values);
ierr += HYPRE_ParCSRMatrixRestoreRow(ParMatrix_, i+MyRowStart_,&num_entries,&indices,&values);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get row of matrix.");
entries = num_entries;
for(int j = 0; j < num_entries; j++){
// Insert column ids from this row into set
new_value = indices[j];
Columns.insert(new_value);
}
}
int NumMyCols = Columns.size();
Teuchos::Array<int> GlobalColIDs; GlobalColIDs.resize(NumMyCols);
std::set<int>::iterator it;
int counter = 0;
for (it = Columns.begin(); it != Columns.end(); it++) {
// Get column ids in order
GlobalColIDs[counter] = *it;
counter = counter + 1;
}
//printf("Proc[%d] Rows from %d to %d, num = %d\n", Comm().MyPID(), MyRowStart_,MyRowEnd_, NumMyRows_);
Epetra_Map RowMap(-1, NumMyRows_, &GlobalRowIDs[0], 0, Comm());
Epetra_Map ColMap(-1, NumMyCols, &GlobalColIDs[0], 0, Comm());
//Need to call SetMaps()
SetMaps(RowMap, ColMap);
// Get an MPI_Comm to create vectors.
// The vectors will be reused in Multiply(), so that they aren't recreated every time.
MPI_Comm comm;
ierr += HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get communicator from Hypre Matrix.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &X_hypre);
ierr += HYPRE_IJVectorSetObjectType(X_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(X_hypre);
ierr += HYPRE_IJVectorAssemble(X_hypre);
ierr += HYPRE_IJVectorGetObject(X_hypre, (void**) &par_x);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre X vector.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &Y_hypre);
ierr += HYPRE_IJVectorSetObjectType(Y_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(Y_hypre);
ierr += HYPRE_IJVectorAssemble(Y_hypre);
ierr += HYPRE_IJVectorGetObject(Y_hypre, (void**) &par_y);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre Y vector.");
x_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) X_hypre));
x_local = hypre_ParVectorLocalVector(x_vec);
y_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) Y_hypre));
y_local = hypre_ParVectorLocalVector(y_vec);
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRPCGCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRPCGDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRPCGSetup;
SolverSolvePtr_ = &HYPRE_ParCSRPCGSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRPCGSetPrecond;
CreateSolver();
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_EuclidCreate;
PrecondDestroyPtr_ = &HYPRE_EuclidDestroy;
PrecondSetupPtr_ = &HYPRE_EuclidSetup;
PrecondSolvePtr_ = &HYPRE_EuclidSolve;
CreatePrecond();
ComputeNumericConstants();
ComputeStructureConstants();
} //EpetraExt_HYPREIJMatrix(Hypre_IJMatrix) Constructor
void solvetriadmatrixwithtrilinos(int& nnz, int& order, int* row,
int* col, double* val, double* rhs, double* solution) {
#else
void solvetriadmatrixwithtrilinos_(int& nnz, int& order, int* row,
int* col, double* val, double* rhs, double* solution) {
#endif
try{
#ifdef _MPI
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int i, j, ierr;
int MyPID = Comm.MyPID();
bool verbose = (MyPID == 0);
Epetra_Map RowMap(order, 0, Comm);
int NumMyElements = RowMap.NumMyElements();
int *MyGlobalElements = new int[NumMyElements];
RowMap.MyGlobalElements(&MyGlobalElements[0]);
#ifdef _MPI
int nPEs;
MPI_Comm_size(MPI_COMM_WORLD, &nPEs);
#endif
int anEst = nnz / order + 1;
Epetra_CrsMatrix A(Copy, RowMap, anEst);
for (j=0; j<nnz; ++j) {
if (RowMap.MyGID(row[j]) ) {
ierr = A.InsertGlobalValues(row[j], 1, &(val[j]), &(col[j]) );
assert(ierr >= 0);
}
}
ierr = A.FillComplete();
assert(ierr == 0);
//-------------------------------------------------------------------------
// RN_20091221: Taking care of the rhs
//-------------------------------------------------------------------------
Epetra_Vector b(RowMap);
// Inserting values into the rhs
double *MyGlobalValues = new double[NumMyElements];
for (j=0; j<NumMyElements; ++j) {
MyGlobalValues[j] = rhs[MyGlobalElements[j] ];
}
ierr = b.ReplaceGlobalValues(NumMyElements, &MyGlobalValues[0],
&MyGlobalElements[0]);
//-------------------------------------------------------------------------
// RN_20091221: Taking care of the solution
//-------------------------------------------------------------------------
Epetra_Vector x(RowMap);
Teuchos::ParameterList paramList;
Teuchos::RCP<Teuchos::ParameterList>
paramList1 = Teuchos::rcp(¶mList, false);
Teuchos::updateParametersFromXmlFile("./strat1.xml", paramList1.get() );
Teuchos::RCP<Teuchos::FancyOStream>
out = Teuchos::VerboseObjectBase::getDefaultOStream();
Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
Teuchos::RCP<Epetra_CrsMatrix> epetraOper = Teuchos::rcp(&A, false);
Teuchos::RCP<Epetra_Vector> epetraRhs = Teuchos::rcp(&b, false);
Teuchos::RCP<Epetra_Vector> epetraSol = Teuchos::rcp(&x, false);
Teuchos::RCP<const Thyra::LinearOpBase<double> >
thyraOper = Thyra::epetraLinearOp(epetraOper);
Teuchos::RCP<Thyra::VectorBase<double> >
thyraRhs = Thyra::create_Vector(epetraRhs, thyraOper->range() );
Teuchos::RCP<Thyra::VectorBase<double> >
thyraSol = Thyra::create_Vector(epetraSol, thyraOper->domain() );
linearSolverBuilder.setParameterList(Teuchos::rcp(¶mList, false) );
Teuchos::RCP<Thyra::LinearOpWithSolveFactoryBase<double> >
lowsFactory = linearSolverBuilder.createLinearSolveStrategy("");
lowsFactory->setOStream(out);
lowsFactory->setVerbLevel(Teuchos::VERB_LOW);
Teuchos::RCP<Thyra::LinearOpWithSolveBase<double> >
lows = Thyra::linearOpWithSolve(*lowsFactory, thyraOper);
Thyra::SolveStatus<double>
status = Thyra::solve(*lows, Thyra::NOTRANS, *thyraRhs, &*thyraSol);
thyraSol = Teuchos::null;
// For debugging =)
// cout << "A: " << A << endl;
// cout << "b: " << b << endl;
// cout << "x: " << x << endl;
// Epetra_Vector temp(RowMap);
// double residualNorm;
// A.Multiply(false, x, temp);
// temp.Update(-1, b, 1);
// temp.Norm2(&residualNorm);
Epetra_LocalMap localMap(order, 0, Comm);
Epetra_Vector xExtra(localMap); // local vector in each processor
// RN_20091218: Create an import map and then import the data to the
// local vector.
Epetra_Import import(localMap, RowMap);
xExtra.Import(x, import, Add);
xExtra.ExtractCopy(solution);
delete[] MyGlobalElements;
delete[] MyGlobalValues;
}
catch (std::exception& e) {
cout << e.what() << endl;
}
catch (string& s) {
cout << s << endl;
}
catch (char *s) {
cout << s << endl;
}
catch (...) {
cout << "Caught unknown exception!" << endl;
}
}
} // extern "C"
int Epetra_FECrsMatrix::GlobalAssemble(const Epetra_Map& domain_map,
const Epetra_Map& range_map,
bool callFillComplete,
Epetra_CombineMode combineMode,
bool save_off_and_reuse_map_exporter)
{
if (Map().Comm().NumProc() < 2 || ignoreNonLocalEntries_) {
if (callFillComplete) {
EPETRA_CHK_ERR( FillComplete(domain_map, range_map) );
}
return(0);
}
std::vector<int_type>& nonlocalRows_var = nonlocalRows<int_type>();
std::vector<std::vector<int_type> >& nonlocalCols_var = nonlocalCols<int_type>();
if (useNonlocalMatrix_) {
tempMat_ = nonlocalMatrix_;
}
else {
//In this method we need to gather all the non-local (overlapping) data
//that's been input on each processor, into the
//non-overlapping distribution defined by the map that 'this' matrix was
//constructed with.
//First build a map that describes our nonlocal data.
//We'll use the arbitrary distribution constructor of Map.
int_type* nlr_ptr = nonlocalRows_var.size() > 0 ? &nonlocalRows_var[0] : 0;
if (sourceMap_ == NULL)
sourceMap_ = new Epetra_Map((int_type) -1, (int) nonlocalRows_var.size(), nlr_ptr,
(int_type) Map().IndexBase64(), Map().Comm());
//If sourceMap has global size 0, then no nonlocal data exists and we can
//skip most of this function.
if (sourceMap_->NumGlobalElements64() < 1) {
if (callFillComplete) {
EPETRA_CHK_ERR( FillComplete(domain_map, range_map) );
}
if (!save_off_and_reuse_map_exporter) {
delete sourceMap_;
sourceMap_ = NULL;
}
return(0);
}
//We also need to build a column-map, containing the columns in our
//nonlocal data. To do that, create a list of all column-indices that
//occur in our nonlocal rows.
bool first_time=!save_off_and_reuse_map_exporter;
if ( colMap_ == NULL ) {
first_time = true;
std::vector<int_type> cols;
for(size_t i=0; i<nonlocalRows_var.size(); ++i) {
for(size_t j=0; j<nonlocalCols_var[i].size(); ++j) {
int_type col = nonlocalCols_var[i][j];
typename std::vector<int_type>::iterator it =
std::lower_bound(cols.begin(), cols.end(), col);
if (it == cols.end() || *it != col) {
cols.insert(it, col);
}
}
}
int_type* cols_ptr = cols.size() > 0 ? &cols[0] : 0;
colMap_ = new Epetra_Map((int_type) -1, (int) cols.size(), cols_ptr,
(int_type) Map().IndexBase64(), Map().Comm());
}
//now we need to create a matrix with sourceMap and colMap, and fill it with
//our nonlocal data so we can then export it to the correct owning processors.
std::vector<int> nonlocalRowLengths(nonlocalRows_var.size());
for(size_t i=0; i<nonlocalRows_var.size(); ++i) {
nonlocalRowLengths[i] = (int) nonlocalCols_var[i].size();
}
int* nlRLptr = nonlocalRowLengths.size()>0 ? &nonlocalRowLengths[0] : NULL;
if ( first_time && tempMat_ == NULL )
tempMat_ = new Epetra_CrsMatrix(Copy, *sourceMap_, *colMap_, nlRLptr);
else
tempMat_->PutScalar(0.);
for(size_t i=0; i<nonlocalRows_var.size(); ++i) {
if ( first_time ) {
EPETRA_CHK_ERR( tempMat_->InsertGlobalValues(nonlocalRows_var[i],
(int) nonlocalCols_var[i].size(),
&nonlocalCoefs_[i][0],
&nonlocalCols_var[i][0]) );
} else {
EPETRA_CHK_ERR( tempMat_->SumIntoGlobalValues(nonlocalRows_var[i],
(int) nonlocalCols_var[i].size(),
&nonlocalCoefs_[i][0],
&nonlocalCols_var[i][0]) );
}
}
if (!save_off_and_reuse_map_exporter) {
delete sourceMap_;
delete colMap_;
sourceMap_ = colMap_ = NULL;
}
//Next we need to make sure the 'indices-are-global' attribute of tempMat's
//graph is set to true, in case this processor doesn't end up calling the
//InsertGlobalValues method...
if (first_time) {
const Epetra_CrsGraph& graph = tempMat_->Graph();
Epetra_CrsGraph& nonconst_graph = const_cast<Epetra_CrsGraph&>(graph);
nonconst_graph.SetIndicesAreGlobal(true);
}
}
//Now we need to call FillComplete on our temp matrix. We need to
//pass a DomainMap and RangeMap, which are not the same as the RowMap
//and ColMap that we constructed the matrix with.
EPETRA_CHK_ERR(tempMat_->FillComplete(domain_map, range_map));
if (exporter_ == NULL)
exporter_ = new Epetra_Export(tempMat_->RowMap(), RowMap());
EPETRA_CHK_ERR(Export(*tempMat_, *exporter_, combineMode));
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete(domain_map, range_map));
}
//now reset the values in our nonlocal data
if (!useNonlocalMatrix_) {
for(size_t i=0; i<nonlocalRows_var.size(); ++i) {
nonlocalCols_var[i].resize(0);
nonlocalCoefs_[i].resize(0);
}
}
if (!save_off_and_reuse_map_exporter) {
delete exporter_;
exporter_ = NULL;
if (!useNonlocalMatrix_)
delete tempMat_;
tempMat_ = NULL;
}
return(0);
}
// ======================================================================
bool TestTriDiVariableBlocking(const Epetra_Comm & Comm) {
// Basically each processor gets this 5x5 block lower-triangular matrix:
//
// [ 2 -1 0 0 0 ;...
// [-1 2 0 0 0 ;...
// [ 0 -1 3 -1 0 ;...
// [ 0 0 -1 3 -1 ;...
// [ 0 0 0 -1 2 ];
//
Epetra_Map RowMap(-1,5,0,Comm); // 5 rows per proc
Epetra_CrsMatrix A(Copy,RowMap,0);
int num_entries;
int indices[5];
double values[5];
int rb = RowMap.GID(0);
/*** Fill RHS / LHS ***/
Epetra_Vector rhs(RowMap), lhs(RowMap), exact_soln(RowMap);
rhs.PutScalar(2.0);
lhs.PutScalar(0.0);
exact_soln.PutScalar(2.0);
/*** Fill Matrix ****/
// Row 0
num_entries=2;
indices[0]=rb; indices[1]=rb+1;
values[0] =2; values[1] =-1;
A.InsertGlobalValues(rb,num_entries,&values[0],&indices[0]);
// Row 1
num_entries=2;
indices[0]=rb; indices[1]=rb+1;
values[0] =-1; values[1] =2;
A.InsertGlobalValues(rb+1,num_entries,&values[0],&indices[0]);
// Row 2
num_entries=3;
indices[0]=rb+1; indices[1]=rb+2; indices[2]=rb+3;
values[0] =-1; values[1] = 3; values[2] =-1;
A.InsertGlobalValues(rb+2,num_entries,&values[0],&indices[0]);
// Row 3
num_entries=3;
indices[0]=rb+2; indices[1]=rb+3; indices[2]=rb+4;
values[0] =-1; values[1] = 3; values[2] =-1;
A.InsertGlobalValues(rb+3,num_entries,&values[0],&indices[0]);
// Row 4
num_entries=2;
indices[0]=rb+3; indices[1]=rb+4;
values[0] =-1; values[1] = 2;
A.InsertGlobalValues(rb+4,num_entries,&values[0],&indices[0]);
A.FillComplete();
/* Setup Block Relaxation */
int PartMap[5]={0,0,1,1,1};
Teuchos::ParameterList ilist;
ilist.set("partitioner: type","user");
ilist.set("partitioner: map",&PartMap[0]);
ilist.set("partitioner: local parts",2);
ilist.set("relaxation: sweeps",1);
ilist.set("relaxation: type","Gauss-Seidel");
Ifpack_BlockRelaxation<Ifpack_TriDiContainer> TDRelax(&A);
TDRelax.SetParameters(ilist);
TDRelax.Initialize();
TDRelax.Compute();
TDRelax.ApplyInverse(rhs,lhs);
double norm;
lhs.Update(1.0,exact_soln,-1.0);
lhs.Norm2(&norm);
if(verbose) cout<<"Variable Block Partitioning Test"<<endl;
if(norm < 1e-14) {
if(verbose) cout << "Test passed" << endl;
return true;
}
else {
if(verbose) cout << "Test failed" << endl;
return false;
}
}
int Epetra_FECrsMatrix::InputNonlocalGlobalValues(int_type row,
int numCols, const int_type* cols,
const double* values,
int mode)
{
if(!RowMap().template GlobalIndicesIsType<int_type>())
throw ReportError("Epetra_FECrsMatrix::InputNonlocalGlobalValues mismatch between argument types (int/long long) and map type.", -1);
// if we already have a nonlocal matrix object, this is easier...
if (useNonlocalMatrix_) {
int err, returncode = 0;
double* valuesptr = (double*)values;
switch(mode) {
case Epetra_FECrsMatrix::SUMINTO:
err = nonlocalMatrix_->SumIntoGlobalValues(row, numCols,
valuesptr, (int_type*)cols);
if (err<0) return(err);
if (err>0) returncode = err;
break;
case Epetra_FECrsMatrix::REPLACE:
err = nonlocalMatrix_->ReplaceGlobalValues(row, numCols,
valuesptr, (int_type*)cols);
if (err<0) return(err);
if (err>0) returncode = err;
break;
case Epetra_FECrsMatrix::INSERT:
err = nonlocalMatrix_->InsertGlobalValues(row, numCols,
valuesptr, (int_type*)cols);
if (err<0) return(err);
if (err>0) returncode = err;
break;
default:
std::cerr << "Epetra_FECrsMatrix: internal error, bad input mode."<< std::endl;
return(-1);
}
return (returncode);
}
int ierr1 = 0, ierr2 = 0;
#ifdef EPETRA_HAVE_OMP
#ifdef EPETRA_HAVE_OMP_NONASSOCIATIVE
#pragma omp critical
#endif
#endif
{
std::vector<int_type>& nonlocalRows_var = nonlocalRows<int_type>();
//find offset of this row in our list of nonlocal rows.
typename std::vector<int_type>::iterator it =
std::lower_bound(nonlocalRows_var.begin(), nonlocalRows_var.end(), row);
int rowoffset = (int) (it - nonlocalRows_var.begin());
if (it == nonlocalRows_var.end() || *it != row) {
ierr1 = InsertNonlocalRow(row, it);
}
for(int i=0; i<numCols; ++i) {
ierr2 = InputNonlocalValue(rowoffset, cols[i], values[i], mode);
}
}
EPETRA_CHK_ERR(ierr1);
EPETRA_CHK_ERR(ierr2);
return(0);
}
int Epetra_FECrsMatrix::InputGlobalValues_RowMajor(
int numRows, const int_type* rows,
int numCols, const int_type* cols,
const double* values,
int mode)
{
if(!RowMap().template GlobalIndicesIsType<int_type>())
throw ReportError("Epetra_FECrsMatrix::InputGlobalValues_RowMajor mismatch between argument types (int/long long) and map type.", -1);
int returncode = 0;
int err = 0;
for(int i=0; i<numRows; ++i) {
double* valuesptr = (double*)values + i*numCols;
int local_row_id = Map().LID(rows[i]);
if (local_row_id >= 0) {
switch(mode) {
case Epetra_FECrsMatrix::SUMINTO:
err = this->Epetra_CrsMatrix::SumIntoGlobalValues(rows[i], numCols,
valuesptr, (int_type*)cols);
if (err<0) return(err);
if (err>0) returncode = err;
break;
case Epetra_FECrsMatrix::REPLACE:
err = this->Epetra_CrsMatrix::ReplaceGlobalValues(rows[i], numCols,
valuesptr, (int_type*)cols);
if (err<0) return(err);
if (err>0) returncode = err;
break;
case Epetra_FECrsMatrix::INSERT:
err = this->Epetra_CrsMatrix::InsertGlobalValues(rows[i], numCols,
valuesptr, (int_type*)cols);
if (err<0) return(err);
if (err>0) returncode = err;
break;
default:
std::cerr << "Epetra_FECrsMatrix: internal error, bad input mode."<< std::endl;
return(-1);
}
}
else {
#ifdef EPETRA_HAVE_OMP
#ifdef EPETRA_HAVE_OMP_NONASSOCIATIVE
if (! ignoreNonLocalEntries_) {
#endif
#endif
err = InputNonlocalGlobalValues(rows[i], numCols, cols,
valuesptr, mode);
if (err<0) return(err);
if (err>0) returncode = err;
#ifdef EPETRA_HAVE_OMP
#ifdef EPETRA_HAVE_OMP_NONASSOCIATIVE
}
#endif
#endif
}
}
return(returncode);
}
//=============================================================================
int Epetra_FastCrsMatrix::Multiply(bool TransA, const Epetra_MultiVector& X, Epetra_MultiVector& Y) const {
//
// This function forms the product Y = A * Y or Y = A' * X
//
if (X.NumVectors()==1 && Y.NumVectors()==1) {
double * xp = (double *) X[0];
double * yp = (double *) Y[0];
Epetra_Vector x(View, X.Map(), xp);
Epetra_Vector y(View, Y.Map(), yp);
return(Multiply(TransA, x, y));
}
if (!Filled()) EPETRA_CHK_ERR(-1); // Matrix must be filled.
int i, j, k;
int * NumEntriesPerRow = NumEntriesPerRow_;
int ** Indices = Indices_;
double ** Values = Values_;
double **Xp = (double**)X.Pointers();
double **Yp = (double**)Y.Pointers();
int NumVectors = X.NumVectors();
int NumMyCols_ = NumMyCols();
// Need to better manage the Import and Export vectors:
// - Need accessor functions
// - Need to make the NumVector match (use a View to do this)
// - Need to look at RightScale and ColSum routines too.
if (!TransA) {
// If we have a non-trivial importer, we must import elements that are permuted or are on other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),NumVectors); // Create import vector if needed
ImportVector_->Import(X, *Importer(), Insert);
Xp = (double**)ImportVector_->Pointers();
}
// If we have a non-trivial exporter, we must export elements that are permuted or belong to other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=NumVectors) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),NumVectors); // Create Export vector if needed
Yp = (double**)ExportVector_->Pointers();
}
// Do actual computation
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (k=0; k<NumVectors; k++) {
double sum = 0.0;
for (j=0; j < NumEntries; j++) sum += RowValues[j] * Xp[k][RowIndices[j]];
Yp[k][i] = sum;
}
}
if (Exporter()!=0) Y.Export(*ExportVector_, *Exporter(), Add); // Fill Y with Values from export vector
}
else { // Transpose operation
// If we have a non-trivial exporter, we must import elements that are permuted or are on other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=NumVectors) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),NumVectors); // Create Export vector if needed
ExportVector_->Import(X, *Exporter(), Insert);
Xp = (double**)ExportVector_->Pointers();
}
// If we have a non-trivial importer, we must export elements that are permuted or belong to other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),NumVectors); // Create import vector if needed
Yp = (double**)ImportVector_->Pointers();
}
// Do actual computation
for (k=0; k<NumVectors; k++)
for (i=0; i < NumMyCols_; i++) Yp[k][i] = 0.0; // Initialize y for transpose multiply
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (k=0; k<NumVectors; k++) {
for (j=0; j < NumEntries; j++) Yp[k][RowIndices[j]] += RowValues[j] * Xp[k][i];
}
}
if (Importer()!=0) Y.Export(*ImportVector_, *Importer(), Add); // Fill Y with Values from export vector
}
UpdateFlops(2*NumVectors*NumGlobalNonzeros64());
return(0);
}
int main(int argc, char *argv[]) {
int i, j, info;
const double one = 1.0;
const double zero = 0.0;
Teuchos::LAPACK<int,double> lapack;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int MyPID = Comm.MyPID();
// Dimension of the matrix
int m = 500;
int n = 100;
// Construct a Map that puts approximately the same number of
// equations on each processor.
Epetra_Map RowMap(m, 0, Comm);
Epetra_Map ColMap(n, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyRowElements = RowMap.NumMyElements();
std::vector<int> MyGlobalRowElements(NumMyRowElements);
RowMap.MyGlobalElements(&MyGlobalRowElements[0]);
/* We are building an m by n matrix with entries
A(i,j) = k*(si)*(tj - 1) if i <= j
= k*(tj)*(si - 1) if i > j
where si = i/(m+1) and tj = j/(n+1) and k = 1/(n+1).
*/
// Create an Epetra_Matrix
Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix(Copy, RowMap, n) );
// Compute coefficients for discrete integral operator
std::vector<double> Values(n);
std::vector<int> Indices(n);
double inv_mp1 = one/(m+1);
double inv_np1 = one/(n+1);
for (i=0; i<n; i++) { Indices[i] = i; }
for (i=0; i<NumMyRowElements; i++) {
//
for (j=0; j<n; j++) {
//
if ( MyGlobalRowElements[i] <= j ) {
Values[j] = inv_np1 * ( (MyGlobalRowElements[i]+one)*inv_mp1 ) * ( (j+one)*inv_np1 - one ); // k*(si)*(tj-1)
}
else {
Values[j] = inv_np1 * ( (j+one)*inv_np1 ) * ( (MyGlobalRowElements[i]+one)*inv_mp1 - one ); // k*(tj)*(si-1)
}
}
info = A->InsertGlobalValues(MyGlobalRowElements[i], n, &Values[0], &Indices[0]);
assert( info==0 );
}
// Finish up
info = A->FillComplete(ColMap, RowMap);
assert( info==0 );
info = A->OptimizeStorage();
assert( info==0 );
A->SetTracebackMode(1); // Shutdown Epetra Warning tracebacks
//************************************
// Start the block Arnoldi iteration
//***********************************
//
// Variables used for the Block Arnoldi Method
//
int nev = 4;
int blockSize = 1;
int numBlocks = 10;
int maxRestarts = 20;
int verbosity = Anasazi::Errors + Anasazi::Warnings + Anasazi::FinalSummary;
double tol = lapack.LAMCH('E');
std::string which = "LM";
//
// Create parameter list to pass into solver
//
Teuchos::ParameterList MyPL;
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Which", which );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Num Blocks", numBlocks );
MyPL.set( "Maximum Restarts", maxRestarts );
MyPL.set( "Convergence Tolerance", tol );
typedef Anasazi::MultiVec<double> MV;
typedef Anasazi::Operator<double> OP;
// Create an Anasazi::EpetraMultiVec for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
Teuchos::RCP<Anasazi::EpetraMultiVec> ivec = Teuchos::rcp( new Anasazi::EpetraMultiVec(ColMap, blockSize) );
ivec->MvRandom();
// Call the constructor for the (A^T*A) operator
Teuchos::RCP<Anasazi::EpetraSymOp> Amat = Teuchos::rcp( new Anasazi::EpetraSymOp(A) );
Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > MyProblem =
Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(Amat, ivec) );
// Inform the eigenproblem that the matrix A is symmetric
MyProblem->setHermitian(true);
// Set the number of eigenvalues requested and the blocksize the solver should use
MyProblem->setNEV( nev );
// Inform the eigenproblem that you are finished passing it information
bool boolret = MyProblem->setProblem();
if (boolret != true) {
if (MyPID == 0) {
cout << "Anasazi::BasicEigenproblem::setProblem() returned with error." << endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
// Initialize the Block Arnoldi solver
Anasazi::BlockKrylovSchurSolMgr<double, MV, OP> MySolverMgr(MyProblem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMgr.solve();
if (returnCode != Anasazi::Converged && MyPID==0) {
cout << "Anasazi::EigensolverMgr::solve() returned unconverged." << endl;
}
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<double,MV> sol = MyProblem->getSolution();
std::vector<Anasazi::Value<double> > evals = sol.Evals;
int numev = sol.numVecs;
if (numev > 0) {
// Compute singular values/vectors and direct residuals.
//
// Compute singular values which are the square root of the eigenvalues
if (MyPID==0) {
cout<<"------------------------------------------------------"<<endl;
cout<<"Computed Singular Values: "<<endl;
cout<<"------------------------------------------------------"<<endl;
}
for (i=0; i<numev; i++) { evals[i].realpart = Teuchos::ScalarTraits<double>::squareroot( evals[i].realpart ); }
//
// Compute left singular vectors : u = Av/sigma
//
std::vector<double> tempnrm(numev), directnrm(numev);
std::vector<int> index(numev);
for (i=0; i<numev; i++) { index[i] = i; }
Anasazi::EpetraMultiVec Av(RowMap,numev), u(RowMap,numev);
Anasazi::EpetraMultiVec* evecs = dynamic_cast<Anasazi::EpetraMultiVec* >(sol.Evecs->CloneViewNonConst( index ));
Teuchos::SerialDenseMatrix<int,double> S(numev,numev);
A->Apply( *evecs, Av );
Av.MvNorm( tempnrm );
for (i=0; i<numev; i++) { S(i,i) = one/tempnrm[i]; };
u.MvTimesMatAddMv( one, Av, S, zero );
//
// Compute direct residuals : || Av - sigma*u ||
//
for (i=0; i<numev; i++) { S(i,i) = evals[i].realpart; }
Av.MvTimesMatAddMv( -one, u, S, one );
Av.MvNorm( directnrm );
if (MyPID==0) {
cout.setf(std::ios_base::right, std::ios_base::adjustfield);
cout<<std::setw(16)<<"Singular Value"
<<std::setw(20)<<"Direct Residual"
<<endl;
cout<<"------------------------------------------------------"<<endl;
for (i=0; i<numev; i++) {
cout<<std::setw(16)<<evals[i].realpart
<<std::setw(20)<<directnrm[i]
<<endl;
}
cout<<"------------------------------------------------------"<<endl;
}
}
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0;
}
int Epetra_FEVbrMatrix::GlobalAssemble(bool callFillComplete)
{
if(Map().Comm().NumProc() < 2 || ignoreNonLocalEntries_) {
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete());
}
return(0);
}
int i;
//In this method we need to gather all the non-local (overlapping) data
//that's been input on each processor, into the
//non-overlapping distribution defined by the map that 'this' matrix was
//constructed with.
//Need to build a map that describes our nonlocal data.
//First, create a list of the sizes (point-rows per block-row) of the
//nonlocal rows we're holding.
int* pointRowsPerNonlocalBlockRow = numNonlocalBlockRows_>0 ?
new int[numNonlocalBlockRows_] : NULL;
for(i=0; i<numNonlocalBlockRows_; ++i) {
pointRowsPerNonlocalBlockRow[i] = nonlocalCoefs_[i][0]->M();
}
//We'll use the arbitrary distribution constructor of BlockMap.
Epetra_BlockMap sourceMap(-1, numNonlocalBlockRows_, nonlocalBlockRows_, // CJ TODO FIXME long long
pointRowsPerNonlocalBlockRow,
RowMap().IndexBase(), RowMap().Comm());
delete [] pointRowsPerNonlocalBlockRow;
//If sourceMap has global size 0, then no nonlocal data exists and we can
//skip most of this function.
if(sourceMap.NumGlobalElements64() < 1) {
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete());
}
return(0);
}
//We also need to build a column-map, containing the columns in our
//nonlocal data. To do that, create a list of all column-indices that
//occur in our nonlocal rows.
int numCols = 0, allocLen = 0;
int* cols = NULL;
int* pointColsPerBlockCol = NULL;
int ptColAllocLen = 0;
int insertPoint = -1;
for(i=0; i<numNonlocalBlockRows_; ++i) {
for(int j=0; j<nonlocalBlockRowLengths_[i]; ++j) {
int col = nonlocalBlockCols_[i][j];
int offset = Epetra_Util_binary_search(col, cols, numCols, insertPoint);
if (offset < 0) {
EPETRA_CHK_ERR( Epetra_Util_insert(col, insertPoint, cols,
numCols, allocLen) );
int tmpNumCols = numCols-1;
EPETRA_CHK_ERR( Epetra_Util_insert(nonlocalCoefs_[i][j]->N(),
insertPoint,
pointColsPerBlockCol,
tmpNumCols, ptColAllocLen) );
}
}
}
Epetra_BlockMap colMap(-1, numCols, cols, // CJ TODO FIXME long long
pointColsPerBlockCol,
RowMap().IndexBase(), RowMap().Comm());
delete [] cols;
delete [] pointColsPerBlockCol;
numCols = 0;
allocLen = 0;
//now we need to create a matrix with sourceMap and colMap, and fill it with
//our nonlocal data so we can then export it to the correct owning
//processors.
Epetra_VbrMatrix tempMat(Copy, sourceMap, colMap, nonlocalBlockRowLengths_);
//Next we need to make sure the 'indices-are-global' attribute of tempMat's
//graph is set to true, in case this processor doesn't end up calling the
//InsertGlobalValues method...
const Epetra_CrsGraph& graph = tempMat.Graph();
Epetra_CrsGraph& nonconst_graph = const_cast<Epetra_CrsGraph&>(graph);
nonconst_graph.SetIndicesAreGlobal(true);
for(i=0; i<numNonlocalBlockRows_; ++i) {
EPETRA_CHK_ERR( tempMat.BeginInsertGlobalValues(nonlocalBlockRows_[i],
nonlocalBlockRowLengths_[i],
nonlocalBlockCols_[i]) );
for(int j=0; j<nonlocalBlockRowLengths_[i]; ++j) {
Epetra_SerialDenseMatrix& subblock = *(nonlocalCoefs_[i][j]);
EPETRA_CHK_ERR( tempMat.SubmitBlockEntry(subblock.A(),
subblock.LDA(),
subblock.M(),
subblock.N()) );
}
EPETRA_CHK_ERR( tempMat.EndSubmitEntries() );
}
//Now we need to call FillComplete on our temp matrix. We need to
//pass a DomainMap and RangeMap, which are not the same as the RowMap
//and ColMap that we constructed the matrix with.
EPETRA_CHK_ERR(tempMat.FillComplete(RowMap(), sourceMap));
//Finally, we're ready to create the exporter and export non-local data to
//the appropriate owning processors.
Epetra_Export exporter(sourceMap, RowMap());
EPETRA_CHK_ERR( Export(tempMat, exporter, Add) );
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete());
}
destroyNonlocalData();
return(0);
}
int main(int argc, char *argv[])
{
int ierr = 0, forierr = 0;
bool debug = false;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
int rank; // My process ID
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
int rank = 0;
Epetra_SerialComm Comm;
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
int verbose_int = verbose ? 1 : 0;
Comm.Broadcast(&verbose_int, 1, 0);
verbose = verbose_int==1 ? true : false;
// char tmp;
// if (rank==0) cout << "Press any key to continue..."<< std::endl;
// if (rank==0) cin >> tmp;
// Comm.Barrier();
Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
int MyPID = Comm.MyPID();
int NumProc = Comm.NumProc();
if(verbose && MyPID==0)
cout << Epetra_Version() << std::endl << std::endl;
if (verbose) cout << "Processor "<<MyPID<<" of "<< NumProc
<< " is alive."<<endl;
bool verbose1 = verbose;
// Redefine verbose to only print on PE 0
if(verbose && rank!=0)
verbose = false;
int NumMyEquations = 10000;
int NumGlobalEquations = (NumMyEquations * NumProc) + EPETRA_MIN(NumProc,3);
if(MyPID < 3)
NumMyEquations++;
// Construct a Map that puts approximately the same Number of equations on each processor
Epetra_Map Map(NumGlobalEquations, NumMyEquations, 0, Comm);
// Get update list and number of local equations from newly created Map
int* MyGlobalElements = new int[Map.NumMyElements()];
Map.MyGlobalElements(MyGlobalElements);
// Create an integer vector NumNz that is used to build the Petra Matrix.
// NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation on this processor
int* NumNz = new int[NumMyEquations];
// We are building a tridiagonal matrix where each row has (-1 2 -1)
// So we need 2 off-diagonal terms (except for the first and last equation)
for (int i = 0; i < NumMyEquations; i++)
if((MyGlobalElements[i] == 0) || (MyGlobalElements[i] == NumGlobalEquations - 1))
NumNz[i] = 1;
else
NumNz[i] = 2;
// Create a Epetra_Matrix
Epetra_CrsMatrix A(Copy, Map, NumNz);
EPETRA_TEST_ERR(A.IndicesAreGlobal(),ierr);
EPETRA_TEST_ERR(A.IndicesAreLocal(),ierr);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double* Values = new double[2];
Values[0] = -1.0;
Values[1] = -1.0;
int* Indices = new int[2];
double two = 2.0;
int NumEntries;
forierr = 0;
for (int i = 0; i < NumMyEquations; i++) {
if(MyGlobalElements[i] == 0) {
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == NumGlobalEquations-1) {
Indices[0] = NumGlobalEquations-2;
NumEntries = 1;
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
forierr += !(A.InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices)==0);
forierr += !(A.InsertGlobalValues(MyGlobalElements[i], 1, &two, MyGlobalElements+i)>0); // Put in the diagonal entry
}
EPETRA_TEST_ERR(forierr,ierr);
int * indexOffsetTmp;
int * indicesTmp;
double * valuesTmp;
// Finish up
EPETRA_TEST_ERR(!(A.IndicesAreGlobal()),ierr);
EPETRA_TEST_ERR(!(A.ExtractCrsDataPointers(indexOffsetTmp, indicesTmp, valuesTmp)==-1),ierr); // Should fail
EPETRA_TEST_ERR(!(A.FillComplete(false)==0),ierr);
EPETRA_TEST_ERR(!(A.ExtractCrsDataPointers(indexOffsetTmp, indicesTmp, valuesTmp)==-1),ierr); // Should fail
EPETRA_TEST_ERR(!(A.IndicesAreLocal()),ierr);
EPETRA_TEST_ERR(A.StorageOptimized(),ierr);
A.OptimizeStorage();
EPETRA_TEST_ERR(!(A.StorageOptimized()),ierr);
EPETRA_TEST_ERR(!(A.ExtractCrsDataPointers(indexOffsetTmp, indicesTmp, valuesTmp)==0),ierr); // Should succeed
const Epetra_CrsGraph & GofA = A.Graph();
EPETRA_TEST_ERR((indicesTmp!=GofA[0] || valuesTmp!=A[0]),ierr); // Extra check to see if operator[] is consistent
EPETRA_TEST_ERR(A.UpperTriangular(),ierr);
EPETRA_TEST_ERR(A.LowerTriangular(),ierr);
int NumMyNonzeros = 3 * NumMyEquations;
if(A.LRID(0) >= 0)
NumMyNonzeros--; // If I own first global row, then there is one less nonzero
if(A.LRID(NumGlobalEquations-1) >= 0)
NumMyNonzeros--; // If I own last global row, then there is one less nonzero
EPETRA_TEST_ERR(check(A, NumMyEquations, NumGlobalEquations, NumMyNonzeros, 3*NumGlobalEquations-2,
MyGlobalElements, verbose),ierr);
forierr = 0;
for (int i = 0; i < NumMyEquations; i++)
forierr += !(A.NumGlobalEntries(MyGlobalElements[i])==NumNz[i]+1);
EPETRA_TEST_ERR(forierr,ierr);
forierr = 0;
for (int i = 0; i < NumMyEquations; i++)
forierr += !(A.NumMyEntries(i)==NumNz[i]+1);
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "\n\nNumEntries function check OK" << std::endl<< std::endl;
EPETRA_TEST_ERR(check_graph_sharing(Comm),ierr);
// Create vectors for Power method
Epetra_Vector q(Map);
Epetra_Vector z(Map);
Epetra_Vector resid(Map);
// variable needed for iteration
double lambda = 0.0;
// int niters = 10000;
int niters = 200;
double tolerance = 1.0e-1;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Iterate
Epetra_Flops flopcounter;
A.SetFlopCounter(flopcounter);
q.SetFlopCounter(A);
z.SetFlopCounter(A);
resid.SetFlopCounter(A);
Epetra_Time timer(Comm);
EPETRA_TEST_ERR(power_method(false, A, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
double elapsed_time = timer.ElapsedTime();
double total_flops = A.Flops() + q.Flops() + z.Flops() + resid.Flops();
double MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "\n\nTotal MFLOPs for first solve = " << MFLOPs << std::endl<< std::endl;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Solve transpose problem
if (verbose) cout << "\n\nUsing transpose of matrix and solving again (should give same result).\n\n"
<< std::endl;
// Iterate
lambda = 0.0;
flopcounter.ResetFlops();
timer.ResetStartTime();
EPETRA_TEST_ERR(power_method(true, A, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
elapsed_time = timer.ElapsedTime();
total_flops = A.Flops() + q.Flops() + z.Flops() + resid.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "\n\nTotal MFLOPs for transpose solve = " << MFLOPs << std::endl<< endl;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Increase diagonal dominance
if (verbose) cout << "\n\nIncreasing the magnitude of first diagonal term and solving again\n\n"
<< endl;
if (A.MyGlobalRow(0)) {
int numvals = A.NumGlobalEntries(0);
double * Rowvals = new double [numvals];
int * Rowinds = new int [numvals];
A.ExtractGlobalRowCopy(0, numvals, numvals, Rowvals, Rowinds); // Get A[0,0]
for (int i=0; i<numvals; i++) if (Rowinds[i] == 0) Rowvals[i] *= 10.0;
A.ReplaceGlobalValues(0, numvals, Rowvals, Rowinds);
delete [] Rowvals;
delete [] Rowinds;
}
// Iterate (again)
lambda = 0.0;
flopcounter.ResetFlops();
timer.ResetStartTime();
EPETRA_TEST_ERR(power_method(false, A, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
elapsed_time = timer.ElapsedTime();
total_flops = A.Flops() + q.Flops() + z.Flops() + resid.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "\n\nTotal MFLOPs for second solve = " << MFLOPs << endl<< endl;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Solve transpose problem
if (verbose) cout << "\n\nUsing transpose of matrix and solving again (should give same result).\n\n"
<< endl;
// Iterate (again)
lambda = 0.0;
flopcounter.ResetFlops();
timer.ResetStartTime();
EPETRA_TEST_ERR(power_method(true, A, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
elapsed_time = timer.ElapsedTime();
total_flops = A.Flops() + q.Flops() + z.Flops() + resid.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "\n\nTotal MFLOPs for tranpose of second solve = " << MFLOPs << endl<< endl;
if (verbose) cout << "\n\n*****Testing constant entry constructor" << endl<< endl;
Epetra_CrsMatrix AA(Copy, Map, 5);
if (debug) Comm.Barrier();
double dble_one = 1.0;
for (int i=0; i< NumMyEquations; i++) AA.InsertGlobalValues(MyGlobalElements[i], 1, &dble_one, MyGlobalElements+i);
// Note: All processors will call the following Insert routines, but only the processor
// that owns it will actually do anything
int One = 1;
if (AA.MyGlobalRow(0)) {
EPETRA_TEST_ERR(!(AA.InsertGlobalValues(0, 0, &dble_one, &One)==0),ierr);
}
else EPETRA_TEST_ERR(!(AA.InsertGlobalValues(0, 1, &dble_one, &One)==-1),ierr);
EPETRA_TEST_ERR(!(AA.FillComplete(false)==0),ierr);
EPETRA_TEST_ERR(AA.StorageOptimized(),ierr);
EPETRA_TEST_ERR(!(AA.UpperTriangular()),ierr);
EPETRA_TEST_ERR(!(AA.LowerTriangular()),ierr);
if (debug) Comm.Barrier();
EPETRA_TEST_ERR(check(AA, NumMyEquations, NumGlobalEquations, NumMyEquations, NumGlobalEquations,
MyGlobalElements, verbose),ierr);
if (debug) Comm.Barrier();
forierr = 0;
for (int i=0; i<NumMyEquations; i++) forierr += !(AA.NumGlobalEntries(MyGlobalElements[i])==1);
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "\n\nNumEntries function check OK" << endl<< endl;
if (debug) Comm.Barrier();
if (verbose) cout << "\n\n*****Testing copy constructor" << endl<< endl;
Epetra_CrsMatrix B(AA);
EPETRA_TEST_ERR(check(B, NumMyEquations, NumGlobalEquations, NumMyEquations, NumGlobalEquations,
MyGlobalElements, verbose),ierr);
forierr = 0;
for (int i=0; i<NumMyEquations; i++) forierr += !(B.NumGlobalEntries(MyGlobalElements[i])==1);
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "\n\nNumEntries function check OK" << endl<< endl;
if (debug) Comm.Barrier();
if (verbose) cout << "\n\n*****Testing local view constructor" << endl<< endl;
Epetra_CrsMatrix BV(View, AA.RowMap(), AA.ColMap(), 0);
forierr = 0;
int* Inds;
double* Vals;
for (int i = 0; i < NumMyEquations; i++) {
forierr += !(AA.ExtractMyRowView(i, NumEntries, Vals, Inds)==0);
forierr += !(BV.InsertMyValues(i, NumEntries, Vals, Inds)==0);
}
BV.FillComplete(false);
EPETRA_TEST_ERR(check(BV, NumMyEquations, NumGlobalEquations, NumMyEquations, NumGlobalEquations,
MyGlobalElements, verbose),ierr);
forierr = 0;
for (int i=0; i<NumMyEquations; i++) forierr += !(BV.NumGlobalEntries(MyGlobalElements[i])==1);
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "\n\nNumEntries function check OK" << endl<< endl;
if (debug) Comm.Barrier();
if (verbose) cout << "\n\n*****Testing post construction modifications" << endl<< endl;
EPETRA_TEST_ERR(!(B.InsertGlobalValues(0, 1, &dble_one, &One)==-2),ierr);
// Release all objects
delete [] NumNz;
delete [] Values;
delete [] Indices;
delete [] MyGlobalElements;
if (verbose1) {
// Test ostream << operator (if verbose1)
// Construct a Map that puts 2 equations on each PE
int NumMyElements1 = 2;
int NumMyEquations1 = NumMyElements1;
int NumGlobalEquations1 = NumMyEquations1*NumProc;
Epetra_Map Map1(-1, NumMyElements1, 0, Comm);
// Get update list and number of local equations from newly created Map
int * MyGlobalElements1 = new int[Map1.NumMyElements()];
Map1.MyGlobalElements(MyGlobalElements1);
// Create an integer vector NumNz that is used to build the Petra Matrix.
// NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation on this processor
int * NumNz1 = new int[NumMyEquations1];
// We are building a tridiagonal matrix where each row has (-1 2 -1)
// So we need 2 off-diagonal terms (except for the first and last equation)
for (int i=0; i<NumMyEquations1; i++)
if (MyGlobalElements1[i]==0 || MyGlobalElements1[i] == NumGlobalEquations1-1)
NumNz1[i] = 1;
else
NumNz1[i] = 2;
// Create a Epetra_Matrix
Epetra_CrsMatrix A1(Copy, Map1, NumNz1);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values1 = new double[2];
Values1[0] = -1.0; Values1[1] = -1.0;
int *Indices1 = new int[2];
double two1 = 2.0;
int NumEntries1;
forierr = 0;
for (int i=0; i<NumMyEquations1; i++)
{
if (MyGlobalElements1[i]==0)
{
Indices1[0] = 1;
NumEntries1 = 1;
}
else if (MyGlobalElements1[i] == NumGlobalEquations1-1)
{
Indices1[0] = NumGlobalEquations1-2;
NumEntries1 = 1;
}
else
{
Indices1[0] = MyGlobalElements1[i]-1;
Indices1[1] = MyGlobalElements1[i]+1;
NumEntries1 = 2;
}
forierr += !(A1.InsertGlobalValues(MyGlobalElements1[i], NumEntries1, Values1, Indices1)==0);
forierr += !(A1.InsertGlobalValues(MyGlobalElements1[i], 1, &two1, MyGlobalElements1+i)>0); // Put in the diagonal entry
}
EPETRA_TEST_ERR(forierr,ierr);
delete [] Indices1;
delete [] Values1;
// Finish up
EPETRA_TEST_ERR(!(A1.FillComplete(false)==0),ierr);
// Test diagonal extraction function
Epetra_Vector checkDiag(Map1);
EPETRA_TEST_ERR(!(A1.ExtractDiagonalCopy(checkDiag)==0),ierr);
forierr = 0;
for (int i=0; i<NumMyEquations1; i++) forierr += !(checkDiag[i]==two1);
EPETRA_TEST_ERR(forierr,ierr);
// Test diagonal replacement method
forierr = 0;
for (int i=0; i<NumMyEquations1; i++) checkDiag[i]=two1*two1;
EPETRA_TEST_ERR(forierr,ierr);
EPETRA_TEST_ERR(!(A1.ReplaceDiagonalValues(checkDiag)==0),ierr);
Epetra_Vector checkDiag1(Map1);
EPETRA_TEST_ERR(!(A1.ExtractDiagonalCopy(checkDiag1)==0),ierr);
forierr = 0;
for (int i=0; i<NumMyEquations1; i++) forierr += !(checkDiag[i]==checkDiag1[i]);
EPETRA_TEST_ERR(forierr,ierr);
if (verbose) cout << "\n\nDiagonal extraction and replacement OK.\n\n" << endl;
double orignorm = A1.NormOne();
EPETRA_TEST_ERR(!(A1.Scale(4.0)==0),ierr);
EPETRA_TEST_ERR(!(A1.NormOne()!=orignorm),ierr);
if (verbose) cout << "\n\nMatrix scale OK.\n\n" << endl;
if (verbose) cout << "\n\nPrint out tridiagonal matrix, each part on each processor.\n\n" << endl;
cout << A1 << endl;
// Release all objects
delete [] NumNz1;
delete [] MyGlobalElements1;
}
if (verbose) cout << "\n\n*****Testing LeftScale and RightScale" << endl << endl;
int NumMyElements2 = 7;
int NumMyRows2 = 1;//This value should not be changed without editing the
// code below.
Epetra_Map RowMap(-1,NumMyRows2,0,Comm);
Epetra_Map ColMap(NumMyElements2,NumMyElements2,0,Comm);
// The DomainMap needs to be different from the ColMap for the test to
// be meaningful.
Epetra_Map DomainMap(NumMyElements2,0,Comm);
int NumMyRangeElements2 = 0;
// We need to distribute the elements differently for the range map also.
if (MyPID % 2 == 0)
NumMyRangeElements2 = NumMyRows2*2; //put elements on even number procs
if (NumProc % 2 == 1 && MyPID == NumProc-1)
NumMyRangeElements2 = NumMyRows2; //If number of procs is odd, put
// the last NumMyElements2 elements on the last proc
Epetra_Map RangeMap(-1,NumMyRangeElements2,0,Comm);
Epetra_CrsMatrix A2(Copy,RowMap,ColMap,NumMyElements2);
double * Values2 = new double[NumMyElements2];
int * Indices2 = new int[NumMyElements2];
for (int i=0; i<NumMyElements2; i++) {
Values2[i] = i+MyPID;
Indices2[i]=i;
}
A2.InsertMyValues(0,NumMyElements2,Values2,Indices2);
A2.FillComplete(DomainMap,RangeMap,false);
Epetra_CrsMatrix A2copy(A2);
double * RowLeftScaleValues = new double[NumMyRows2];
double * ColRightScaleValues = new double[NumMyElements2];
int RowLoopLength = RowMap.MaxMyGID()-RowMap.MinMyGID()+1;
for (int i=0; i<RowLoopLength; i++)
RowLeftScaleValues[i] = (i + RowMap.MinMyGID() ) % 2 + 1;
// For the column map, all procs own all elements
for (int i=0; i<NumMyElements2;i++)
ColRightScaleValues[i] = i % 2 + 1;
int RangeLoopLength = RangeMap.MaxMyGID()-RangeMap.MinMyGID()+1;
double * RangeLeftScaleValues = new double[RangeLoopLength];
int DomainLoopLength = DomainMap.MaxMyGID()-DomainMap.MinMyGID()+1;
double * DomainRightScaleValues = new double[DomainLoopLength];
for (int i=0; i<RangeLoopLength; i++)
RangeLeftScaleValues[i] = 1.0/((i + RangeMap.MinMyGID() ) % 2 + 1);
for (int i=0; i<DomainLoopLength;i++)
DomainRightScaleValues[i] = 1.0/((i + DomainMap.MinMyGID() ) % 2 + 1);
Epetra_Vector xRow(View,RowMap,RowLeftScaleValues);
Epetra_Vector xCol(View,ColMap,ColRightScaleValues);
Epetra_Vector xRange(View,RangeMap,RangeLeftScaleValues);
Epetra_Vector xDomain(View,DomainMap,DomainRightScaleValues);
double A2infNorm = A2.NormInf();
double A2oneNorm = A2.NormOne();
if (verbose1) cout << A2;
EPETRA_TEST_ERR(A2.LeftScale(xRow),ierr);
double A2infNorm1 = A2.NormInf();
double A2oneNorm1 = A2.NormOne();
bool ScalingBroke = false;
if (A2infNorm1>2*A2infNorm||A2infNorm1<A2infNorm) {
EPETRA_TEST_ERR(-31,ierr);
ScalingBroke = true;
}
if (A2oneNorm1>2*A2oneNorm||A2oneNorm1<A2oneNorm) {
EPETRA_TEST_ERR(-32,ierr);
ScalingBroke = true;
}
if (verbose1) cout << A2;
EPETRA_TEST_ERR(A2.RightScale(xCol),ierr);
double A2infNorm2 = A2.NormInf();
double A2oneNorm2 = A2.NormOne();
if (A2infNorm2>=2*A2infNorm1||A2infNorm2<=A2infNorm1) {
EPETRA_TEST_ERR(-33,ierr);
ScalingBroke = true;
}
if (A2oneNorm2>2*A2oneNorm1||A2oneNorm2<=A2oneNorm1) {
EPETRA_TEST_ERR(-34,ierr);
ScalingBroke = true;
}
if (verbose1) cout << A2;
EPETRA_TEST_ERR(A2.RightScale(xDomain),ierr);
double A2infNorm3 = A2.NormInf();
double A2oneNorm3 = A2.NormOne();
// The last two scaling ops cancel each other out
if (A2infNorm3!=A2infNorm1) {
EPETRA_TEST_ERR(-35,ierr)
ScalingBroke = true;
}
if (A2oneNorm3!=A2oneNorm1) {
EPETRA_TEST_ERR(-36,ierr)
ScalingBroke = true;
}
if (verbose1) cout << A2;
EPETRA_TEST_ERR(A2.LeftScale(xRange),ierr);
double A2infNorm4 = A2.NormInf();
double A2oneNorm4 = A2.NormOne();
// The 4 scaling ops all cancel out
if (A2infNorm4!=A2infNorm) {
EPETRA_TEST_ERR(-37,ierr)
ScalingBroke = true;
}
if (A2oneNorm4!=A2oneNorm) {
EPETRA_TEST_ERR(-38,ierr)
ScalingBroke = true;
}
//
// Now try changing the values underneath and make sure that
// telling one process about the change causes NormInf() and
// NormOne() to recompute the norm on all processes.
//
double *values;
int num_my_rows = A2.NumMyRows() ;
int num_entries;
for ( int i=0 ; i< num_my_rows; i++ ) {
EPETRA_TEST_ERR( A2.ExtractMyRowView( i, num_entries, values ), ierr );
for ( int j = 0 ; j <num_entries; j++ ) {
values[j] *= 2.0;
}
}
if ( MyPID == 0 )
A2.SumIntoGlobalValues( 0, 0, 0, 0 ) ;
double A2infNorm5 = A2.NormInf();
double A2oneNorm5 = A2.NormOne();
if (A2infNorm5!=2.0 * A2infNorm4) {
EPETRA_TEST_ERR(-39,ierr)
ScalingBroke = true;
}
if (A2oneNorm5!= 2.0 * A2oneNorm4) {
EPETRA_TEST_ERR(-40,ierr)
ScalingBroke = true;
}
//
// Restore the values underneath
//
for ( int i=0 ; i< num_my_rows; i++ ) {
EPETRA_TEST_ERR( A2.ExtractMyRowView( i, num_entries, values ), ierr );
for ( int j = 0 ; j <num_entries; j++ ) {
values[j] /= 2.0;
}
}
if (verbose1) cout << A2;
if (ScalingBroke) {
if (verbose) cout << endl << "LeftScale and RightScale tests FAILED" << endl << endl;
}
else {
if (verbose) cout << endl << "LeftScale and RightScale tests PASSED" << endl << endl;
}
Comm.Barrier();
if (verbose) cout << "\n\n*****Testing InvRowMaxs and InvColMaxs" << endl << endl;
if (verbose1) cout << A2 << endl;
EPETRA_TEST_ERR(A2.InvRowMaxs(xRow),ierr);
EPETRA_TEST_ERR(A2.InvRowMaxs(xRange),ierr);
if (verbose1) cout << xRow << endl << xRange << endl;
if (verbose) cout << "\n\n*****Testing InvRowSums and InvColSums" << endl << endl;
bool InvSumsBroke = false;
// Works!
EPETRA_TEST_ERR(A2.InvRowSums(xRow),ierr);
if (verbose1) cout << xRow;
EPETRA_TEST_ERR(A2.LeftScale(xRow),ierr);
float A2infNormFloat = A2.NormInf();
if (verbose1) cout << A2 << endl;
if (fabs(1.0-A2infNormFloat) > 1.e-5) {
EPETRA_TEST_ERR(-41,ierr);
InvSumsBroke = true;
}
// Works
int expectedcode = 1;
if (Comm.NumProc()>1) expectedcode = 0;
EPETRA_TEST_ERR(!(A2.InvColSums(xDomain)==expectedcode),ierr); // This matrix has a single row, the first column has a zero, so a warning is issued.
if (verbose1) cout << xDomain << endl;
EPETRA_TEST_ERR(A2.RightScale(xDomain),ierr);
float A2oneNormFloat2 = A2.NormOne();
if (verbose1) cout << A2;
if (fabs(1.0-A2oneNormFloat2)>1.e-5) {
EPETRA_TEST_ERR(-42,ierr)
InvSumsBroke = true;
}
// Works!
EPETRA_TEST_ERR(A2.InvRowSums(xRange),ierr);
if (verbose1) cout << xRange;
EPETRA_TEST_ERR(A2.LeftScale(xRange),ierr);
float A2infNormFloat2 = A2.NormInf(); // We use a float so that rounding error
// will not prevent the sum from being 1.0.
if (verbose1) cout << A2;
if (fabs(1.0-A2infNormFloat2)>1.e-5) {
cout << "InfNorm should be = 1, but InfNorm = " << A2infNormFloat2 << endl;
EPETRA_TEST_ERR(-43,ierr);
InvSumsBroke = true;
}
// Doesn't work - may not need this test because column ownership is not unique
/* EPETRA_TEST_ERR(A2.InvColSums(xCol),ierr);
cout << xCol;
EPETRA_TEST_ERR(A2.RightScale(xCol),ierr);
float A2oneNormFloat = A2.NormOne();
cout << A2;
if (fabs(1.0-A2oneNormFloat)>1.e-5) {
EPETRA_TEST_ERR(-44,ierr);
InvSumsBroke = true;
}
*/
delete [] ColRightScaleValues;
delete [] DomainRightScaleValues;
if (verbose) cout << "Begin partial sum testing." << endl;
// Test with a matrix that has partial sums for a subset of the rows
// on multiple processors. (Except for the serial case, of course.)
int NumMyRows3 = 2; // Changing this requires further changes below
int * myGlobalElements = new int[NumMyRows3];
for (int i=0; i<NumMyRows3; i++) myGlobalElements[i] = MyPID+i;
Epetra_Map RowMap3(NumProc*2, NumMyRows3, myGlobalElements, 0, Comm);
int NumMyElements3 = 5;
Epetra_CrsMatrix A3(Copy, RowMap3, NumMyElements3);
double * Values3 = new double[NumMyElements3];
int * Indices3 = new int[NumMyElements3];
for (int i=0; i < NumMyElements3; i++) {
Values3[i] = (int) (MyPID + (i+1));
Indices3[i]=i;
}
for (int i=0; i<NumMyRows3; i++) {
A3.InsertGlobalValues(myGlobalElements[i],NumMyElements3,Values3,Indices3);
}
Epetra_Map RangeMap3(NumProc+1, 0, Comm);
Epetra_Map DomainMap3(NumMyElements3, 0, Comm);
EPETRA_TEST_ERR(A3.FillComplete(DomainMap3, RangeMap3,false),ierr);
if (verbose1) cout << A3;
Epetra_Vector xRange3(RangeMap3,false);
Epetra_Vector xDomain3(DomainMap3,false);
EPETRA_TEST_ERR(A3.InvRowSums(xRange3),ierr);
if (verbose1) cout << xRange3;
EPETRA_TEST_ERR(A3.LeftScale(xRange3),ierr);
float A3infNormFloat = A3.NormInf();
if (verbose1) cout << A3;
if (1.0!=A3infNormFloat) {
cout << "InfNorm should be = 1, but InfNorm = " << A3infNormFloat <<endl;
EPETRA_TEST_ERR(-61,ierr);
InvSumsBroke = true;
}
// we want to take the transpose of our matrix and fill in different values.
int NumMyColumns3 = NumMyRows3;
Epetra_Map ColMap3cm(RowMap3);
Epetra_Map RowMap3cm(A3.ColMap());
Epetra_CrsMatrix A3cm(Copy,RowMap3cm,ColMap3cm,NumProc+1);
double *Values3cm = new double[NumMyColumns3];
int * Indices3cm = new int[NumMyColumns3];
for (int i=0; i<NumMyColumns3; i++) {
Values3cm[i] = MyPID + i + 1;
Indices3cm[i]= i + MyPID;
}
for (int ii=0; ii<NumMyElements3; ii++) {
A3cm.InsertGlobalValues(ii, NumMyColumns3, Values3cm, Indices3cm);
}
// The DomainMap and the RangeMap from the last test will work fine for
// the RangeMap and DomainMap, respectively, but I will make copies to
// avaoid confusion when passing what looks like a DomainMap where we
// need a RangeMap and vice vera.
Epetra_Map RangeMap3cm(DomainMap3);
Epetra_Map DomainMap3cm(RangeMap3);
EPETRA_TEST_ERR(A3cm.FillComplete(DomainMap3cm,RangeMap3cm),ierr);
if (verbose1) cout << A3cm << endl;
// Again, we can copy objects from the last example.
//Epetra_Vector xRange3cm(xDomain3); //Don't use at this time
Epetra_Vector xDomain3cm(DomainMap3cm,false);
EPETRA_TEST_ERR(A3cm.InvColSums(xDomain3cm),ierr);
if (verbose1) cout << xDomain3cm << endl;
EPETRA_TEST_ERR(A3cm.RightScale(xDomain3cm),ierr);
float A3cmOneNormFloat = A3cm.NormOne();
if (verbose1) cout << A3cm << endl;
if (1.0!=A3cmOneNormFloat) {
cout << "OneNorm should be = 1, but OneNorm = " << A3cmOneNormFloat << endl;
EPETRA_TEST_ERR(-62,ierr);
InvSumsBroke = true;
}
if (verbose) cout << "End partial sum testing" << endl;
if (verbose) cout << "Begin replicated testing" << endl;
// We will now view the shared row as a repliated row, rather than one
// that has partial sums of its entries on mulitple processors.
// We will reuse much of the data used for the partial sum tesitng.
Epetra_Vector xRow3(RowMap3,false);
Epetra_CrsMatrix A4(Copy, RowMap3, NumMyElements3);
for (int ii=0; ii < NumMyElements3; ii++) {
Values3[ii] = (int)((ii*.6)+1.0);
}
for (int ii=0; ii<NumMyRows3; ii++) {
A4.InsertGlobalValues(myGlobalElements[ii],NumMyElements3,Values3,Indices3);
}
EPETRA_TEST_ERR(A4.FillComplete(DomainMap3, RangeMap3,false),ierr);
if (verbose1) cout << A4 << endl;
// The next two lines should be expanded into a verifiable test.
EPETRA_TEST_ERR(A4.InvRowMaxs(xRow3),ierr);
EPETRA_TEST_ERR(A4.InvRowMaxs(xRange3),ierr);
if (verbose1) cout << xRow3 << xRange3;
EPETRA_TEST_ERR(A4.InvRowSums(xRow3),ierr);
if (verbose1) cout << xRow3;
EPETRA_TEST_ERR(A4.LeftScale(xRow3),ierr);
float A4infNormFloat = A4.NormInf();
if (verbose1) cout << A4;
if (2.0!=A4infNormFloat && NumProc != 1) {
if (verbose1) cout << "InfNorm should be = 2 (because one column is replicated on two processors and NormOne() does not handle replication), but InfNorm = " << A4infNormFloat <<endl;
EPETRA_TEST_ERR(-63,ierr);
InvSumsBroke = true;
}
else if (1.0!=A4infNormFloat && NumProc == 1) {
if (verbose1) cout << "InfNorm should be = 1, but InfNorm = " << A4infNormFloat <<endl;
EPETRA_TEST_ERR(-63,ierr);
InvSumsBroke = true;
}
Epetra_Vector xCol3cm(ColMap3cm,false);
Epetra_CrsMatrix A4cm(Copy, RowMap3cm, ColMap3cm, NumProc+1);
//Use values from A3cm
for (int ii=0; ii<NumMyElements3; ii++) {
A4cm.InsertGlobalValues(ii,NumMyColumns3,Values3cm,Indices3cm);
}
EPETRA_TEST_ERR(A4cm.FillComplete(DomainMap3cm, RangeMap3cm,false),ierr);
if (verbose1) cout << A4cm << endl;
// The next two lines should be expanded into a verifiable test.
EPETRA_TEST_ERR(A4cm.InvColMaxs(xCol3cm),ierr);
EPETRA_TEST_ERR(A4cm.InvColMaxs(xDomain3cm),ierr);
if (verbose1) cout << xCol3cm << xDomain3cm;
EPETRA_TEST_ERR(A4cm.InvColSums(xCol3cm),ierr);
if (verbose1) cout << xCol3cm << endl;
EPETRA_TEST_ERR(A4cm.RightScale(xCol3cm),ierr);
float A4cmOneNormFloat = A4cm.NormOne();
if (verbose1) cout << A4cm << endl;
if (2.0!=A4cmOneNormFloat && NumProc != 1) {
if (verbose1) cout << "OneNorm should be = 2 (because one column is replicated on two processors and NormOne() does not handle replication), but OneNorm = " << A4cmOneNormFloat << endl;
EPETRA_TEST_ERR(-64,ierr);
InvSumsBroke = true;
}
else if (1.0!=A4cmOneNormFloat && NumProc == 1) {
if (verbose1) cout << "OneNorm should be = 1, but OneNorm = " << A4infNormFloat <<endl;
EPETRA_TEST_ERR(-64,ierr);
InvSumsBroke = true;
}
if (verbose) cout << "End replicated testing" << endl;
if (InvSumsBroke) {
if (verbose) cout << endl << "InvRowSums tests FAILED" << endl << endl;
}
else
if (verbose) cout << endl << "InvRowSums tests PASSED" << endl << endl;
A3cm.PutScalar(2.0);
int nnz_A3cm = A3cm.Graph().NumGlobalNonzeros();
double check_frobnorm = sqrt(nnz_A3cm*4.0);
double frobnorm = A3cm.NormFrobenius();
bool frobnorm_test_failed = false;
if (fabs(check_frobnorm-frobnorm) > 5.e-5) {
frobnorm_test_failed = true;
}
if (frobnorm_test_failed) {
if (verbose) std::cout << "Frobenius-norm test FAILED."<<std::endl;
EPETRA_TEST_ERR(-65, ierr);
}
delete [] Values2;
delete [] Indices2;
delete [] myGlobalElements;
delete [] Values3;
delete [] Indices3;
delete [] Values3cm;
delete [] Indices3cm;
delete [] RangeLeftScaleValues;
delete [] RowLeftScaleValues;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
/* end main
*/
return ierr ;
}
int main(int argc, char *argv[])
{
int ierr = 0;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc, &argv);
int rank; // My process ID
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
int rank = 0;
Epetra_SerialComm Comm;
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
int verbose_int = verbose ? 1 : 0;
Comm.Broadcast(&verbose_int, 1, 0);
verbose = verbose_int==1 ? true : false;
Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
int MyPID = Comm.MyPID();
int NumProc = Comm.NumProc();
if(verbose && MyPID==0)
std::cout << Epetra_Version() << std::endl << std::endl;
if (verbose) std::cout << "Processor "<<MyPID<<" of "<< NumProc
<< " is alive."<< std::endl;
// unused: bool verbose1 = verbose;
// Redefine verbose to only print on PE 0
if(verbose && rank!=0)
verbose = false;
if (verbose) std::cout << "Test the memory management system of the class CrsMatrix (memory leak, invalid free)" << std::endl;
//
// Test 1: code initially proposed to illustrate bug #5499
//
if(Comm.NumProc() == 1) { // this is a sequential test
if (verbose) std::cout << "* Using Copy, ColMap, Variable number of indices per row and Static profile (cf. bug #5499)." << std::endl;
// Row Map
Epetra_Map RowMap(2LL, 0LL, Comm);
// ColMap
std::vector<long long> colids(2);
colids[0]=0;
colids[1]=1;
Epetra_Map ColMap(-1LL, 2, &colids[0], 0LL, Comm);
// NumEntriesPerRow
std::vector<int> NumEntriesPerRow(2);
NumEntriesPerRow[0]=2;
NumEntriesPerRow[1]=2;
// Test
Epetra_CrsMatrix A(Copy, RowMap, ColMap, &NumEntriesPerRow[0], true);
// Bug #5499 shows up because InsertGlobalValues() is not called (CrsMatrix::Values_ not allocated but freed)
A.FillComplete();
}
//
// Test 1 Bis: same as Test1, but without ColMap and variable number of indices per row. Does not seems to matter
//
if(Comm.NumProc() == 1) { // this is a sequential test
if (verbose) std::cout << "* Using Copy, Fixed number of indices per row and Static profile" << std::endl;
Epetra_Map RowMap(2LL, 0LL, Comm);
// Test
Epetra_CrsMatrix A(Copy, RowMap, 1, true);
// Bug #5499 shows up because InsertGlobalValues() is not called (CrsMatrix::Values_ not allocated but freed)
A.FillComplete();
}
//
// Test 2: same as Test 1 Bis but with one call to InsertGlobalValues.
//
if(Comm.NumProc() == 1) {
if (verbose) std::cout << "* Using Copy, Fixed number of indices per row and Static profile + InsertGlobalValues()." << std::endl;
Epetra_Map RowMap(2LL, 0LL, Comm);
// Test
Epetra_CrsMatrix A(Copy, RowMap, 1, true);
std::vector<long long> Indices(1);
std::vector<double> Values(1);
Values[0] = 2;
Indices[0] = 0;
A.InsertGlobalValues(0, 1, &Values[0], &Indices[0]); // Memory leak if CrsMatrix::Values not freed
A.FillComplete();
}
//
// Test 3: check if the patch is not introducing some obvious regression
//
if(Comm.NumProc() == 1) {
if (verbose) std::cout << "* Using Copy, Fixed number of indices per row and Dynamic profile" << std::endl;
Epetra_Map RowMap(2LL, 0LL, Comm);
// Test
Epetra_CrsMatrix A(Copy, RowMap, 1, false);
A.FillComplete();
}
//
// Test 4: idem but with one call to InsertGlobalValues.
//
if(Comm.NumProc() == 1) {
if (verbose) std::cout << "* Using Copy, Fixed number of indices per row and Dynamic profile + InsertGlobalValues()." << std::endl;
Epetra_Map RowMap(2LL, 0LL, Comm);
// Test
Epetra_CrsMatrix A(Copy, RowMap, 1, false);
std::vector<long long> Indices(1);
std::vector<double> Values(1);
Values[0] = 2;
Indices[0] = 0;
A.InsertGlobalValues(0, 1, &Values[0], &Indices[0]);
A.FillComplete();
}
if(Comm.NumProc() == 1) {
if (verbose) std::cout << "* Using Copy, Static Graph()." << std::endl;
Epetra_Map RowMap(1LL, 0LL, Comm);
// Test
Epetra_CrsGraph G(Copy, RowMap, 1);
std::vector<long long> Indices(1);
Indices[0] = 0;
G.InsertGlobalIndices(0, 1, &Indices[0]);
G.FillComplete();
Epetra_CrsMatrix A(Copy, G);
std::vector<double> Values(1);
Values[0] = 2;
A.ReplaceGlobalValues(0, 1, &Values[0], &Indices[0]);
A.FillComplete();
double norminf = A.NormInf();
if (verbose) std::cout << "** Inf Norm of Matrix = " << norminf << "." << std::endl;
std::cout << A << std::endl;
}
if(Comm.NumProc() == 1) {
if (verbose) std::cout << "* Using Copy, Fixed number of indices per row and static profile + InsertGlobalValues() for a single row." << std::endl;
Epetra_Map RowMap(1LL, 0LL, Comm);
// Test
Epetra_CrsMatrix A(Copy, RowMap, 1, true);
std::vector<long long> Indices(1);
std::vector<double> Values(1);
Values[0] = 2;
Indices[0] = 0;
A.InsertGlobalValues(0, 1, &Values[0], &Indices[0]);
A.FillComplete();
}
/*
if (bool) {
if (verbose) std::cout << std::endl << "tests FAILED" << std::endl << std::endl;
}
else {*/
if (verbose) std::cout << std::endl << "tests PASSED" << std::endl << std::endl;
/* } */
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return ierr;
}
int test_bug2554(Epetra_Comm& Comm, bool verbose)
{
//This function contains code submitted by Joe Young to
//expose bug 2554. The bug has now been fixed, so this
//function executes without problem. It will be kept as
//a regression test.
// Construct maps that do not have consecutive indices
int RowIndices[3];
if (Comm.MyPID() == 0) {
RowIndices[0] = 1;
RowIndices[1] = 2;
RowIndices[2] = 3;
} else {
RowIndices[0] = 4;
RowIndices[1] = 5;
RowIndices[2] = 6;
}
Epetra_Map RowMap(-1, 3, RowIndices, 0, Comm);
// Construct a graph with two entries per line
Epetra_CrsGraph Graph(Copy, RowMap, 2);
for (int i = 0; i < RowMap.NumMyElements(); i++) {
int ig = RowIndices[i];
Graph.InsertGlobalIndices(ig, 1, &ig);
}
Graph.FillComplete();
// Make some matrix out of this
Epetra_FECrsMatrix *Matrix=new Epetra_FECrsMatrix(Copy, Graph);
// Fill it up with ones
Matrix->PutScalar(1.0);
// Create a rhs and lhs
Epetra_Vector *rhs=new Epetra_Vector(RowMap);
Epetra_Vector *lhs=new Epetra_Vector(RowMap);
rhs->PutScalar(2.0);
lhs->PutScalar(0.0);
// Create a solver and problem;
AztecOO *solver=new AztecOO();
Epetra_LinearProblem *problem=new Epetra_LinearProblem();
// Load the problem into the solver
problem->SetOperator(Matrix);
problem->SetRHS(rhs);
problem->SetLHS(lhs);
solver->SetProblem(*problem, true);
// Set some options
solver->SetAztecOption(AZ_solver,AZ_cg);
solver->SetAztecOption(AZ_precond,AZ_ls);
solver->SetAztecOption(AZ_poly_ord,9);
// Solve the problem
solver->Iterate(50,1e-12);
// Delete the matrix, lhs, rhs
delete Matrix;
delete lhs;
delete rhs;
/* Somehow, C++ reallocates objects in the same location where
they were previously allocated. So, we need to trick it. If we
don't do this, the error will not appear. */
int *dummy=new int[1000];
double *dummy2=new double[1000];
// Reallocate all of them
Matrix=new Epetra_FECrsMatrix(Copy, Graph);
rhs=new Epetra_Vector(RowMap);
lhs=new Epetra_Vector(RowMap);
Matrix->PutScalar(1.0);
rhs->PutScalar(2.0);
lhs->PutScalar(0.0);
// Load the problem into the solver
problem->SetOperator(Matrix);
problem->SetRHS(rhs);
problem->SetLHS(lhs);
//For the following call to SetProblem, we must pass 'true' for
//the optional bool argument which would otherwise default to false.
//Passing 'true' forces it to internally reset the preconditioner.
solver->SetProblem(*problem, true);
// Solve the problem
solver->Iterate(50,1e-12);
// Clean up some memory
solver->UnsetLHSRHS(); // Make sure this function works
delete problem;
delete solver;
delete [] dummy;
delete [] dummy2;
delete Matrix;
delete lhs;
delete rhs;
return(0);
}
