int TUnpackWithOwningPIDsCount(const Epetra_CrsMatrix& SourceMatrix, int NumSameIDs, int NumRemoteIDs, const int * RemoteLIDs, int NumPermuteIDs, const int *PermuteToLIDs, const int *PermuteFromLIDs, int LenImports, char* Imports) { int i,nnz=0; int SizeofIntType = (int)sizeof(int_type); // SameIDs: Always first, always in the same place for(i=0; i<NumSameIDs; i++) nnz+=SourceMatrix.NumMyEntries(i); // PermuteIDs: Still local, but reordered for(i=0; i<NumPermuteIDs; i++) nnz += SourceMatrix.NumMyEntries(PermuteFromLIDs[i]); // RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; // General version int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); for(i=0; i<NumRemoteIDs; i++) { nnz += NumEntries; if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); } } } return nnz; }
bool CrsMatrixInfo( const Epetra_CrsMatrix & A, ostream & os ) { int MyPID = A.Comm().MyPID(); // take care that matrix is already trasformed bool IndicesAreGlobal = A.IndicesAreGlobal(); if( IndicesAreGlobal == true ) { if( MyPID == 0 ) { os << "WARNING : matrix must be transformed to local\n"; os << " before calling CrsMatrixInfo\n"; os << " Now returning...\n"; } return false; } int NumGlobalRows = A.NumGlobalRows(); int NumGlobalNonzeros = A.NumGlobalNonzeros(); int NumGlobalCols = A.NumGlobalCols(); double NormInf = A.NormInf(); double NormOne = A.NormOne(); int NumGlobalDiagonals = A.NumGlobalDiagonals(); int GlobalMaxNumEntries = A.GlobalMaxNumEntries(); int IndexBase = A.IndexBase(); bool StorageOptimized = A.StorageOptimized(); bool LowerTriangular = A.LowerTriangular(); bool UpperTriangular = A.UpperTriangular(); bool NoDiagonal = A.NoDiagonal(); // these variables identifies quantities I have to compute, // since not provided by Epetra_CrsMatrix double MyFrobeniusNorm( 0.0 ), FrobeniusNorm( 0.0 ); double MyMinElement( DBL_MAX ), MinElement( DBL_MAX ); double MyMaxElement( DBL_MIN ), MaxElement( DBL_MIN ); double MyMinAbsElement( DBL_MAX ), MinAbsElement( DBL_MAX ); double MyMaxAbsElement( 0.0 ), MaxAbsElement( 0.0 ); int NumMyRows = A.NumMyRows(); int * NzPerRow = new int[NumMyRows]; int Row; // iterator on rows int Col; // iterator on cols int MaxNumEntries = A.MaxNumEntries(); double * Values = new double[MaxNumEntries]; int * Indices = new int[MaxNumEntries]; double Element, AbsElement; // generic nonzero element and its abs value int NumEntries; double * Diagonal = new double [NumMyRows]; // SumOffDiagonal is the sum of absolute values for off-diagonals double * SumOffDiagonal = new double [NumMyRows]; for( Row=0 ; Row<NumMyRows ; ++Row ) { SumOffDiagonal[Row] = 0.0; } int * IsDiagonallyDominant = new int [NumMyRows]; int GlobalRow; // cycle over all matrix elements for( Row=0 ; Row<NumMyRows ; ++Row ) { GlobalRow = A.GRID(Row); NzPerRow[Row] = A.NumMyEntries(Row); A.ExtractMyRowCopy(Row,NzPerRow[Row],NumEntries,Values,Indices); for( Col=0 ; Col<NumEntries ; ++Col ) { Element = Values[Col]; AbsElement = abs(Element); if( Element<MyMinElement ) MyMinElement = Element; if( Element>MyMaxElement ) MyMaxElement = Element; if( AbsElement<MyMinAbsElement ) MyMinAbsElement = AbsElement; if( AbsElement>MyMaxAbsElement ) MyMaxAbsElement = AbsElement; if( Indices[Col] == Row ) Diagonal[Row] = Element; else SumOffDiagonal[Row] += abs(Element); MyFrobeniusNorm += pow(Element,2); } } // analise storage per row int MyMinNzPerRow( NumMyRows ), MinNzPerRow( NumMyRows ); int MyMaxNzPerRow( 0 ), MaxNzPerRow( 0 ); for( Row=0 ; Row<NumMyRows ; ++Row ) { if( NzPerRow[Row]<MyMinNzPerRow ) MyMinNzPerRow=NzPerRow[Row]; if( NzPerRow[Row]>MyMaxNzPerRow ) MyMaxNzPerRow=NzPerRow[Row]; } // a test to see if matrix is diagonally-dominant int MyDiagonalDominance( 0 ), DiagonalDominance( 0 ); int MyWeakDiagonalDominance( 0 ), WeakDiagonalDominance( 0 ); for( Row=0 ; Row<NumMyRows ; ++Row ) { if( abs(Diagonal[Row])>SumOffDiagonal[Row] ) ++MyDiagonalDominance; else if( abs(Diagonal[Row])==SumOffDiagonal[Row] ) ++MyWeakDiagonalDominance; } // reduction operations A.Comm().SumAll(&MyFrobeniusNorm, &FrobeniusNorm, 1); A.Comm().MinAll(&MyMinElement, &MinElement, 1); A.Comm().MaxAll(&MyMaxElement, &MaxElement, 1); A.Comm().MinAll(&MyMinAbsElement, &MinAbsElement, 1); A.Comm().MaxAll(&MyMaxAbsElement, &MaxAbsElement, 1); A.Comm().MinAll(&MyMinNzPerRow, &MinNzPerRow, 1); A.Comm().MaxAll(&MyMaxNzPerRow, &MaxNzPerRow, 1); A.Comm().SumAll(&MyDiagonalDominance, &DiagonalDominance, 1); A.Comm().SumAll(&MyWeakDiagonalDominance, &WeakDiagonalDominance, 1); // free memory delete Values; delete Indices; delete Diagonal; delete SumOffDiagonal; delete IsDiagonallyDominant; delete NzPerRow; // simply no output for MyPID>0, only proc 0 write on os if( MyPID != 0 ) return true; os << "*** general Information about the matrix\n"; os << "Number of Global Rows = " << NumGlobalRows << endl; os << "Number of Global Cols = " << NumGlobalCols << endl; os << "is the matrix square = " << ((NumGlobalRows==NumGlobalCols)?"yes":"no") << endl; os << "||A||_\\infty = " << NormInf << endl; os << "||A||_1 = " << NormOne << endl; os << "||A||_F = " << sqrt(FrobeniusNorm) << endl; os << "Number of nonzero diagonal entries = " << NumGlobalDiagonals << "( " << 1.0* NumGlobalDiagonals/NumGlobalRows*100 << " %)\n"; os << "Nonzero per row : min = " << MinNzPerRow << " average = " << 1.0*NumGlobalNonzeros/NumGlobalRows << " max = " << MaxNzPerRow << endl; os << "Maximum number of nonzero elements/row = " << GlobalMaxNumEntries << endl; os << "min( a_{i,j} ) = " << MinElement << endl; os << "max( a_{i,j} ) = " << MaxElement << endl; os << "min( abs(a_{i,j}) ) = " << MinAbsElement << endl; os << "max( abs(a_{i,j}) ) = " << MaxAbsElement << endl; os << "Number of diagonal dominant rows = " << DiagonalDominance << " (" << 100.0*DiagonalDominance/NumGlobalRows << " % of total)\n"; os << "Number of weakly diagonal dominant rows = " << WeakDiagonalDominance << " (" << 100.0*WeakDiagonalDominance/NumGlobalRows << " % of total)\n"; os << "*** Information about the Trilinos storage\n"; os << "Base Index = " << IndexBase << endl; os << "is storage optimized = " << ((StorageOptimized==true)?"yes":"no") << endl; os << "are indices global = " << ((IndicesAreGlobal==true)?"yes":"no") << endl; os << "is matrix lower triangular = " << ((LowerTriangular==true)?"yes":"no") << endl; os << "is matrix upper triangular = " << ((UpperTriangular==true)?"yes":"no") << endl; os << "are there diagonal entries = " << ((NoDiagonal==false)?"yes":"no") << endl; return true; }
bool FiniteDifferenceColoringWithUpdate::differenceProbe(const Epetra_Vector& x, Epetra_CrsMatrix& jac,const Epetra_MapColoring& colors){ // Allocate space for perturbation, get column version of x for scaling Epetra_Vector xp(x); Epetra_Vector *xcol; int N=jac.NumMyRows(); if(jac.ColMap().SameAs(x.Map())) xcol=const_cast<Epetra_Vector*>(&x); else{ xcol=new Epetra_Vector(jac.ColMap(),true);//zeros out by default xcol->Import(x,*jac.Importer(),InsertAdd); } // Counters for probing diagnostics double tmp,probing_error_lower_bound=0.0,jc_norm=0.0; // Grab coloring info (being very careful to ignore color 0) int Ncolors=colors.MaxNumColors()+1; int num_c0_global,num_c0_local=colors.NumElementsWithColor(0); colors.Comm().MaxAll(&num_c0_local,&num_c0_global,1); if(num_c0_global>0) Ncolors--; if(Ncolors==0) return false; // Pointers for Matrix Info int entries, *indices; double *values; // NTS: Fix me if ( diffType == Centered ) exit(1); double scaleFactor = 1.0; if ( diffType == Backward ) scaleFactor = -1.0; // Compute RHS at initial solution computeF(x,fo,NOX::Epetra::Interface::Required::FD_Res); /* Probing, vector by vector since computeF does not have a MultiVector interface */ // Assume that anything with Color 0 gets ignored. for(int j=1;j<Ncolors;j++){ xp=x; for(int i=0;i<N;i++){ if(colors[i]==j) xp[i] += scaleFactor*(alpha*abs(x[i])+beta); } computeF(xp, fp, NOX::Epetra::Interface::Required::FD_Res); // Do the subtraction to estimate the Jacobian (w/o including step length) Jc.Update(1.0, fp, -1.0, fo, 0.0); // Relative error in probing if(use_probing_diags){ Jc.Norm2(&tmp); jc_norm+=tmp*tmp; } for(int i=0;i<N;i++){ // Skip for uncolored row/columns, else update entries if(colors[i]==0) continue; jac.ExtractMyRowView(i,entries,values,indices); for(int k=0;k<jac.NumMyEntries(i);k++){ if(colors[indices[k]]==j){ values[k]=Jc[i] / (scaleFactor*(alpha*abs((*xcol)[indices[k]])+beta)); // If probing diagnostics are on, zero out the entries as they are used if(use_probing_diags) Jc[i]=0.0; break;// Only one value per row... } } } if(use_probing_diags){ Jc.Norm2(&tmp); probing_error_lower_bound+=tmp*tmp; } } // If diagnostics are requested, output Frobenius norm lower bound if(use_probing_diags && !x.Comm().MyPID()) printf("Probing Error Lower Bound (Frobenius) abs = %6.4e rel = %6.4e\n",sqrt(probing_error_lower_bound),sqrt(probing_error_lower_bound)/sqrt(jc_norm)); // Cleanup if(!jac.ColMap().SameAs(x.Map())) delete xcol; return true; }
int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix, int NumSameIDs, int NumRemoteIDs, const int * RemoteLIDs, int NumPermuteIDs, const int *PermuteToLIDs, const int *PermuteFromLIDs, int LenImports, char* Imports, int TargetNumRows, int TargetNumNonzeros, int * CSR_rowptr, int_type * CSR_colind, double * CSR_vals, const std::vector<int> &SourcePids, std::vector<int> &TargetPids) { // What we really need to know is where in the CSR arrays each row should start (aka the rowptr). // We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and // (c) Having each row copied into the right colind / values locations. // From Epetra_CrsMatrix UnpackAndCombine(): // Each segment of Exports will be filled by a packed row of information for each row as follows: // 1st int: GRID of row where GRID is the global row ID for the source matrix // next int: NumEntries, Number of indices in row. // next NumEntries: The actual indices for the row. int i,j,rv; int N = TargetNumRows; int mynnz = TargetNumNonzeros; int MyPID = SourceMatrix.Comm().MyPID(); int SizeofIntType = sizeof(int_type); // Zero the rowptr for(i=0; i<N+1; i++) CSR_rowptr[i]=0; // SameIDs: Always first, always in the same place for(i=0; i<NumSameIDs; i++) CSR_rowptr[i]=SourceMatrix.NumMyEntries(i); // PermuteIDs: Still local, but reordered for(i=0; i<NumPermuteIDs; i++) CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]); // RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); for(i=0; i<NumRemoteIDs; i++) { CSR_rowptr[RemoteLIDs[i]] += NumEntries; if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); } } } // If multiple procs contribute to a row; std::vector<int> NewStartRow(N+1); // Turn row length into a real CSR_rowptr int last_len = CSR_rowptr[0]; CSR_rowptr[0] = 0; for(i=1; i<N+1; i++){ int new_len = CSR_rowptr[i]; CSR_rowptr[i] = last_len + CSR_rowptr[i-1]; NewStartRow[i] = CSR_rowptr[i]; last_len = new_len; } // Preseed TargetPids with -1 for local if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz); TargetPids.assign(mynnz,-1); // Grab pointers for SourceMatrix int * Source_rowptr, * Source_colind; double * Source_vals; rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals); if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers"); // SameIDs: Copy the data over for(i=0; i<NumSameIDs; i++) { int FromRow = Source_rowptr[i]; int ToRow = CSR_rowptr[i]; NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i]; for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) { CSR_vals[ToRow + j - FromRow] = Source_vals[j]; CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]); TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1; } } // PermuteIDs: Copy the data over for(i=0; i<NumPermuteIDs; i++) { int FromLID = PermuteFromLIDs[i]; int FromRow = Source_rowptr[FromLID]; int ToRow = CSR_rowptr[PermuteToLIDs[i]]; NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID]; for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) { CSR_vals[ToRow + j - FromRow] = Source_vals[j]; CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]); TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1; } } // RemoteIDs: Loop structure following UnpackAndCombine if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); double* valptr = dintptr + IntSize; for (i=0; i<NumRemoteIDs; i++) { int ToLID = RemoteLIDs[i]; int StartRow = NewStartRow[ToLID]; NewStartRow[ToLID]+=NumEntries; double * values = valptr; int_type * Indices = intptr + 2; for(j=0; j<NumEntries; j++){ CSR_vals[StartRow + j] = values[j]; CSR_colind[StartRow + j] = Indices[2*j]; TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1; } if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); valptr = dintptr + IntSize; } } } return 0; }