int TUnpackWithOwningPIDsCount(const Epetra_CrsMatrix& SourceMatrix,
int NumSameIDs,
int NumRemoteIDs,
const int * RemoteLIDs,
int NumPermuteIDs,
const int *PermuteToLIDs,
const int *PermuteFromLIDs,
int LenImports,
char* Imports)
{
int i,nnz=0;
int SizeofIntType = (int)sizeof(int_type);
// SameIDs: Always first, always in the same place
for(i=0; i<NumSameIDs; i++)
nnz+=SourceMatrix.NumMyEntries(i);
// PermuteIDs: Still local, but reordered
for(i=0; i<NumPermuteIDs; i++)
nnz += SourceMatrix.NumMyEntries(PermuteFromLIDs[i]);
// RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
// General version
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
for(i=0; i<NumRemoteIDs; i++) {
nnz += NumEntries;
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
}
}
}
return nnz;
}
bool CrsMatrixInfo( const Epetra_CrsMatrix & A,
ostream & os )
{
int MyPID = A.Comm().MyPID();
// take care that matrix is already trasformed
bool IndicesAreGlobal = A.IndicesAreGlobal();
if( IndicesAreGlobal == true ) {
if( MyPID == 0 ) {
os << "WARNING : matrix must be transformed to local\n";
os << " before calling CrsMatrixInfo\n";
os << " Now returning...\n";
}
return false;
}
int NumGlobalRows = A.NumGlobalRows();
int NumGlobalNonzeros = A.NumGlobalNonzeros();
int NumGlobalCols = A.NumGlobalCols();
double NormInf = A.NormInf();
double NormOne = A.NormOne();
int NumGlobalDiagonals = A.NumGlobalDiagonals();
int GlobalMaxNumEntries = A.GlobalMaxNumEntries();
int IndexBase = A.IndexBase();
bool StorageOptimized = A.StorageOptimized();
bool LowerTriangular = A.LowerTriangular();
bool UpperTriangular = A.UpperTriangular();
bool NoDiagonal = A.NoDiagonal();
// these variables identifies quantities I have to compute,
// since not provided by Epetra_CrsMatrix
double MyFrobeniusNorm( 0.0 ), FrobeniusNorm( 0.0 );
double MyMinElement( DBL_MAX ), MinElement( DBL_MAX );
double MyMaxElement( DBL_MIN ), MaxElement( DBL_MIN );
double MyMinAbsElement( DBL_MAX ), MinAbsElement( DBL_MAX );
double MyMaxAbsElement( 0.0 ), MaxAbsElement( 0.0 );
int NumMyRows = A.NumMyRows();
int * NzPerRow = new int[NumMyRows];
int Row; // iterator on rows
int Col; // iterator on cols
int MaxNumEntries = A.MaxNumEntries();
double * Values = new double[MaxNumEntries];
int * Indices = new int[MaxNumEntries];
double Element, AbsElement; // generic nonzero element and its abs value
int NumEntries;
double * Diagonal = new double [NumMyRows];
// SumOffDiagonal is the sum of absolute values for off-diagonals
double * SumOffDiagonal = new double [NumMyRows];
for( Row=0 ; Row<NumMyRows ; ++Row ) {
SumOffDiagonal[Row] = 0.0;
}
int * IsDiagonallyDominant = new int [NumMyRows];
int GlobalRow;
// cycle over all matrix elements
for( Row=0 ; Row<NumMyRows ; ++Row ) {
GlobalRow = A.GRID(Row);
NzPerRow[Row] = A.NumMyEntries(Row);
A.ExtractMyRowCopy(Row,NzPerRow[Row],NumEntries,Values,Indices);
for( Col=0 ; Col<NumEntries ; ++Col ) {
Element = Values[Col];
AbsElement = abs(Element);
if( Element<MyMinElement ) MyMinElement = Element;
if( Element>MyMaxElement ) MyMaxElement = Element;
if( AbsElement<MyMinAbsElement ) MyMinAbsElement = AbsElement;
if( AbsElement>MyMaxAbsElement ) MyMaxAbsElement = AbsElement;
if( Indices[Col] == Row ) Diagonal[Row] = Element;
else
SumOffDiagonal[Row] += abs(Element);
MyFrobeniusNorm += pow(Element,2);
}
}
// analise storage per row
int MyMinNzPerRow( NumMyRows ), MinNzPerRow( NumMyRows );
int MyMaxNzPerRow( 0 ), MaxNzPerRow( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( NzPerRow[Row]<MyMinNzPerRow ) MyMinNzPerRow=NzPerRow[Row];
if( NzPerRow[Row]>MyMaxNzPerRow ) MyMaxNzPerRow=NzPerRow[Row];
}
// a test to see if matrix is diagonally-dominant
int MyDiagonalDominance( 0 ), DiagonalDominance( 0 );
int MyWeakDiagonalDominance( 0 ), WeakDiagonalDominance( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( abs(Diagonal[Row])>SumOffDiagonal[Row] )
++MyDiagonalDominance;
else if( abs(Diagonal[Row])==SumOffDiagonal[Row] )
++MyWeakDiagonalDominance;
}
// reduction operations
A.Comm().SumAll(&MyFrobeniusNorm, &FrobeniusNorm, 1);
A.Comm().MinAll(&MyMinElement, &MinElement, 1);
A.Comm().MaxAll(&MyMaxElement, &MaxElement, 1);
A.Comm().MinAll(&MyMinAbsElement, &MinAbsElement, 1);
A.Comm().MaxAll(&MyMaxAbsElement, &MaxAbsElement, 1);
A.Comm().MinAll(&MyMinNzPerRow, &MinNzPerRow, 1);
A.Comm().MaxAll(&MyMaxNzPerRow, &MaxNzPerRow, 1);
A.Comm().SumAll(&MyDiagonalDominance, &DiagonalDominance, 1);
A.Comm().SumAll(&MyWeakDiagonalDominance, &WeakDiagonalDominance, 1);
// free memory
delete Values;
delete Indices;
delete Diagonal;
delete SumOffDiagonal;
delete IsDiagonallyDominant;
delete NzPerRow;
// simply no output for MyPID>0, only proc 0 write on os
if( MyPID != 0 ) return true;
os << "*** general Information about the matrix\n";
os << "Number of Global Rows = " << NumGlobalRows << endl;
os << "Number of Global Cols = " << NumGlobalCols << endl;
os << "is the matrix square = " <<
((NumGlobalRows==NumGlobalCols)?"yes":"no") << endl;
os << "||A||_\\infty = " << NormInf << endl;
os << "||A||_1 = " << NormOne << endl;
os << "||A||_F = " << sqrt(FrobeniusNorm) << endl;
os << "Number of nonzero diagonal entries = "
<< NumGlobalDiagonals
<< "( " << 1.0* NumGlobalDiagonals/NumGlobalRows*100
<< " %)\n";
os << "Nonzero per row : min = " << MinNzPerRow
<< " average = " << 1.0*NumGlobalNonzeros/NumGlobalRows
<< " max = " << MaxNzPerRow << endl;
os << "Maximum number of nonzero elements/row = "
<< GlobalMaxNumEntries << endl;
os << "min( a_{i,j} ) = " << MinElement << endl;
os << "max( a_{i,j} ) = " << MaxElement << endl;
os << "min( abs(a_{i,j}) ) = " << MinAbsElement << endl;
os << "max( abs(a_{i,j}) ) = " << MaxAbsElement << endl;
os << "Number of diagonal dominant rows = " << DiagonalDominance
<< " (" << 100.0*DiagonalDominance/NumGlobalRows << " % of total)\n";
os << "Number of weakly diagonal dominant rows = "
<< WeakDiagonalDominance
<< " (" << 100.0*WeakDiagonalDominance/NumGlobalRows << " % of total)\n";
os << "*** Information about the Trilinos storage\n";
os << "Base Index = " << IndexBase << endl;
os << "is storage optimized = "
<< ((StorageOptimized==true)?"yes":"no") << endl;
os << "are indices global = "
<< ((IndicesAreGlobal==true)?"yes":"no") << endl;
os << "is matrix lower triangular = "
<< ((LowerTriangular==true)?"yes":"no") << endl;
os << "is matrix upper triangular = "
<< ((UpperTriangular==true)?"yes":"no") << endl;
os << "are there diagonal entries = "
<< ((NoDiagonal==false)?"yes":"no") << endl;
return true;
}
bool FiniteDifferenceColoringWithUpdate::differenceProbe(const Epetra_Vector& x, Epetra_CrsMatrix& jac,const Epetra_MapColoring& colors){
// Allocate space for perturbation, get column version of x for scaling
Epetra_Vector xp(x);
Epetra_Vector *xcol;
int N=jac.NumMyRows();
if(jac.ColMap().SameAs(x.Map()))
xcol=const_cast<Epetra_Vector*>(&x);
else{
xcol=new Epetra_Vector(jac.ColMap(),true);//zeros out by default
xcol->Import(x,*jac.Importer(),InsertAdd);
}
// Counters for probing diagnostics
double tmp,probing_error_lower_bound=0.0,jc_norm=0.0;
// Grab coloring info (being very careful to ignore color 0)
int Ncolors=colors.MaxNumColors()+1;
int num_c0_global,num_c0_local=colors.NumElementsWithColor(0);
colors.Comm().MaxAll(&num_c0_local,&num_c0_global,1);
if(num_c0_global>0) Ncolors--;
if(Ncolors==0) return false;
// Pointers for Matrix Info
int entries, *indices;
double *values;
// NTS: Fix me
if ( diffType == Centered ) exit(1);
double scaleFactor = 1.0;
if ( diffType == Backward )
scaleFactor = -1.0;
// Compute RHS at initial solution
computeF(x,fo,NOX::Epetra::Interface::Required::FD_Res);
/* Probing, vector by vector since computeF does not have a MultiVector interface */
// Assume that anything with Color 0 gets ignored.
for(int j=1;j<Ncolors;j++){
xp=x;
for(int i=0;i<N;i++){
if(colors[i]==j)
xp[i] += scaleFactor*(alpha*abs(x[i])+beta);
}
computeF(xp, fp, NOX::Epetra::Interface::Required::FD_Res);
// Do the subtraction to estimate the Jacobian (w/o including step length)
Jc.Update(1.0, fp, -1.0, fo, 0.0);
// Relative error in probing
if(use_probing_diags){
Jc.Norm2(&tmp);
jc_norm+=tmp*tmp;
}
for(int i=0;i<N;i++){
// Skip for uncolored row/columns, else update entries
if(colors[i]==0) continue;
jac.ExtractMyRowView(i,entries,values,indices);
for(int k=0;k<jac.NumMyEntries(i);k++){
if(colors[indices[k]]==j){
values[k]=Jc[i] / (scaleFactor*(alpha*abs((*xcol)[indices[k]])+beta));
// If probing diagnostics are on, zero out the entries as they are used
if(use_probing_diags) Jc[i]=0.0;
break;// Only one value per row...
}
}
}
if(use_probing_diags){
Jc.Norm2(&tmp);
probing_error_lower_bound+=tmp*tmp;
}
}
// If diagnostics are requested, output Frobenius norm lower bound
if(use_probing_diags && !x.Comm().MyPID()) printf("Probing Error Lower Bound (Frobenius) abs = %6.4e rel = %6.4e\n",sqrt(probing_error_lower_bound),sqrt(probing_error_lower_bound)/sqrt(jc_norm));
// Cleanup
if(!jac.ColMap().SameAs(x.Map()))
delete xcol;
return true;
}
int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix,
int NumSameIDs,
int NumRemoteIDs,
const int * RemoteLIDs,
int NumPermuteIDs,
const int *PermuteToLIDs,
const int *PermuteFromLIDs,
int LenImports,
char* Imports,
int TargetNumRows,
int TargetNumNonzeros,
int * CSR_rowptr,
int_type * CSR_colind,
double * CSR_vals,
const std::vector<int> &SourcePids,
std::vector<int> &TargetPids)
{
// What we really need to know is where in the CSR arrays each row should start (aka the rowptr).
// We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and
// (c) Having each row copied into the right colind / values locations.
// From Epetra_CrsMatrix UnpackAndCombine():
// Each segment of Exports will be filled by a packed row of information for each row as follows:
// 1st int: GRID of row where GRID is the global row ID for the source matrix
// next int: NumEntries, Number of indices in row.
// next NumEntries: The actual indices for the row.
int i,j,rv;
int N = TargetNumRows;
int mynnz = TargetNumNonzeros;
int MyPID = SourceMatrix.Comm().MyPID();
int SizeofIntType = sizeof(int_type);
// Zero the rowptr
for(i=0; i<N+1; i++) CSR_rowptr[i]=0;
// SameIDs: Always first, always in the same place
for(i=0; i<NumSameIDs; i++)
CSR_rowptr[i]=SourceMatrix.NumMyEntries(i);
// PermuteIDs: Still local, but reordered
for(i=0; i<NumPermuteIDs; i++)
CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]);
// RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
for(i=0; i<NumRemoteIDs; i++) {
CSR_rowptr[RemoteLIDs[i]] += NumEntries;
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
}
}
}
// If multiple procs contribute to a row;
std::vector<int> NewStartRow(N+1);
// Turn row length into a real CSR_rowptr
int last_len = CSR_rowptr[0];
CSR_rowptr[0] = 0;
for(i=1; i<N+1; i++){
int new_len = CSR_rowptr[i];
CSR_rowptr[i] = last_len + CSR_rowptr[i-1];
NewStartRow[i] = CSR_rowptr[i];
last_len = new_len;
}
// Preseed TargetPids with -1 for local
if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz);
TargetPids.assign(mynnz,-1);
// Grab pointers for SourceMatrix
int * Source_rowptr, * Source_colind;
double * Source_vals;
rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals);
if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers");
// SameIDs: Copy the data over
for(i=0; i<NumSameIDs; i++) {
int FromRow = Source_rowptr[i];
int ToRow = CSR_rowptr[i];
NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i];
for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// PermuteIDs: Copy the data over
for(i=0; i<NumPermuteIDs; i++) {
int FromLID = PermuteFromLIDs[i];
int FromRow = Source_rowptr[FromLID];
int ToRow = CSR_rowptr[PermuteToLIDs[i]];
NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID];
for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// RemoteIDs: Loop structure following UnpackAndCombine
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
double* valptr = dintptr + IntSize;
for (i=0; i<NumRemoteIDs; i++) {
int ToLID = RemoteLIDs[i];
int StartRow = NewStartRow[ToLID];
NewStartRow[ToLID]+=NumEntries;
double * values = valptr;
int_type * Indices = intptr + 2;
for(j=0; j<NumEntries; j++){
CSR_vals[StartRow + j] = values[j];
CSR_colind[StartRow + j] = Indices[2*j];
TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1;
}
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
valptr = dintptr + IntSize;
}
}
}
return 0;
}
