BlockCrsMatrix::BlockCrsMatrix( const Epetra_RowMatrix & BaseMatrix, const vector< vector<long long> > & RowStencil, const vector<long long> & RowIndices, const Epetra_Comm & GlobalComm ) : Epetra_CrsMatrix( Copy, *(BlockUtility::GenerateBlockGraph( BaseMatrix, RowStencil, RowIndices, GlobalComm )) ), BaseGraph_( Copy, BaseMatrix.RowMatrixRowMap(), 1 ), //Junk to satisfy constructor RowStencil_LL_( RowStencil ), RowIndices_LL_( RowIndices ), ROffset_(BlockUtility::CalculateOffset64(BaseMatrix.RowMatrixRowMap())), COffset_(BlockUtility::CalculateOffset64(BaseMatrix.RowMatrixColMap())) { }
int RowMatrixToHandle(FILE * handle, const Epetra_RowMatrix & A) { #ifndef EPETRA_NO_32BIT_GLOBAL_INDICES if(A.RowMatrixRowMap().GlobalIndicesInt()) { return RowMatrixToHandle<int>(handle, A); } else #endif #ifndef EPETRA_NO_64BIT_GLOBAL_INDICES if(A.RowMatrixRowMap().GlobalIndicesLongLong()) { return RowMatrixToHandle<long long>(handle, A); } else #endif throw "EpetraExt::RowMatrixToHandle: GlobalIndices type unknown"; }
int compute_graph_metrics(const Epetra_RowMatrix &matrix, Isorropia::Epetra::CostDescriber &costs, double &myGoalWeight, double &balance, int &numCuts, double &cutWgt, double &cutn, double &cutl) { const Epetra_Map &rmap = matrix.RowMatrixRowMap(); const Epetra_Map &cmap = matrix.RowMatrixColMap(); int maxEdges = matrix.MaxNumEntries(); std::vector<std::vector<int> > myRows(rmap.NumMyElements()); if (maxEdges > 0){ int numEdges = 0; int *nborLID = new int [maxEdges]; double *tmp = new double [maxEdges]; for (int i=0; i<rmap.NumMyElements(); i++){ matrix.ExtractMyRowCopy(i, maxEdges, numEdges, tmp, nborLID); std::vector<int> cols(numEdges); for (int j=0; j<numEdges; j++){ cols[j] = nborLID[j]; } myRows[i] = cols; } delete [] nborLID; delete [] tmp; } return compute_graph_metrics(rmap, cmap, myRows, costs, myGoalWeight, balance, numCuts, cutWgt, cutn, cutl); }
void BlockCrsMatrix::LoadBlock(const Epetra_RowMatrix & BaseMatrix, const int Row, const int Col) { if(Epetra_CrsMatrix::RowMatrixRowMap().GlobalIndicesInt() && BaseMatrix.RowMatrixRowMap().GlobalIndicesInt()) return TLoadBlock<int>(BaseMatrix, Row, Col); else throw "EpetraExt::BlockCrsMatrix::LoadBlock: Global indices not int"; }
void BlockCrsMatrix::LoadBlock(const Epetra_RowMatrix & BaseMatrix, const long long Row, const long long Col) { if(Epetra_CrsMatrix::RowMatrixRowMap().GlobalIndicesLongLong() && BaseMatrix.RowMatrixRowMap().GlobalIndicesLongLong()) return TLoadBlock<long long>(BaseMatrix, Row, Col); else throw "EpetraExt::BlockCrsMatrix::LoadBlock: Global indices not long long"; }
void BlockCrsMatrix::SumIntoGlobalBlock(double alpha, const Epetra_RowMatrix & BaseMatrix, const int Row, const int Col) { if(Epetra_CrsMatrix::RowMatrixRowMap().GlobalIndicesInt() && BaseMatrix.RowMatrixRowMap().GlobalIndicesInt()) return TSumIntoGlobalBlock<int>(alpha, BaseMatrix, Row, Col); else throw "EpetraExt::BlockCrsMatrix::SumIntoGlobalBlock: Global indices not int"; }
int writeRowMatrix(FILE * handle, const Epetra_RowMatrix & A) { long long numRows_LL = A.NumGlobalRows64(); if(numRows_LL > std::numeric_limits<int>::max()) throw "EpetraExt::writeRowMatrix: numRows_LL > std::numeric_limits<int>::max()"; int numRows = static_cast<int>(numRows_LL); Epetra_Map rowMap = A.RowMatrixRowMap(); Epetra_Map colMap = A.RowMatrixColMap(); const Epetra_Comm & comm = rowMap.Comm(); long long ioffset = 1 - rowMap.IndexBase64(); // Matlab indices start at 1 long long joffset = 1 - colMap.IndexBase64(); // Matlab indices start at 1 if (comm.MyPID()!=0) { if (A.NumMyRows()!=0) {EPETRA_CHK_ERR(-1);} if (A.NumMyCols()!=0) {EPETRA_CHK_ERR(-1);} } else { if (numRows!=A.NumMyRows()) {EPETRA_CHK_ERR(-1);} Epetra_SerialDenseVector values(A.MaxNumEntries()); Epetra_IntSerialDenseVector indices(A.MaxNumEntries()); for (int i=0; i<numRows; i++) { long long I = rowMap.GID64(i) + ioffset; int numEntries; if (A.ExtractMyRowCopy(i, values.Length(), numEntries, values.Values(), indices.Values())!=0) {EPETRA_CHK_ERR(-1);} for (int j=0; j<numEntries; j++) { long long J = colMap.GID64(indices[j]) + joffset; double val = values[j]; fprintf(handle, "%lld %lld %22.16e\n", I, J, val); } } } return(0); }
void BlockCrsMatrix::SumIntoGlobalBlock(double alpha, const Epetra_RowMatrix & BaseMatrix, const long long Row, const long long Col) { if(Epetra_CrsMatrix::RowMatrixRowMap().GlobalIndicesLongLong() && BaseMatrix.RowMatrixRowMap().GlobalIndicesLongLong()) return TSumIntoGlobalBlock<long long>(alpha, BaseMatrix, Row, Col); else throw "EpetraExt::BlockCrsMatrix::SumIntoGlobalBlock: Global indices not long long"; }
int RowMatrixToHandle(FILE * handle, const Epetra_RowMatrix & A) { Epetra_Map map = A.RowMatrixRowMap(); const Epetra_Comm & comm = map.Comm(); int numProc = comm.NumProc(); if (numProc==1 || !A.Map().DistributedGlobal()) writeRowMatrix(handle, A); else { int numRows = map.NumMyElements(); Epetra_Map allGidsMap((int_type) -1, numRows, (int_type) 0,comm); typename Epetra_GIDTypeVector<int_type>::impl allGids(allGidsMap); for (int i=0; i<numRows; i++) allGids[i] = (int_type) map.GID64(i); // Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A. int numChunks = numProc; int stripSize = allGids.GlobalLength64()/numChunks; int remainder = allGids.GlobalLength64()%numChunks; int curStart = 0; int curStripSize = 0; typename Epetra_GIDTypeSerialDenseVector<int_type>::impl importGidList; if (comm.MyPID()==0) importGidList.Size(stripSize+1); // Set size of vector to max needed for (int i=0; i<numChunks; i++) { if (comm.MyPID()==0) { // Only PE 0 does this part curStripSize = stripSize; if (i<remainder) curStripSize++; // handle leftovers for (int j=0; j<curStripSize; j++) importGidList[j] = j + curStart; curStart += curStripSize; } // The following import map will be non-trivial only on PE 0. if (comm.MyPID()>0) assert(curStripSize==0); Epetra_Map importGidMap(-1, curStripSize, importGidList.Values(), 0, comm); Epetra_Import gidImporter(importGidMap, allGidsMap); typename Epetra_GIDTypeVector<int_type>::impl importGids(importGidMap); if (importGids.Import(allGids, gidImporter, Insert)!=0) {EPETRA_CHK_ERR(-1); } // importGids now has a list of GIDs for the current strip of matrix rows. // Use these values to build another importer that will get rows of the matrix. // The following import map will be non-trivial only on PE 0. Epetra_Map importMap(-1, importGids.MyLength(), importGids.Values(), map.IndexBase64(), comm); Epetra_Import importer(importMap, map); Epetra_CrsMatrix importA(Copy, importMap, 0); if (importA.Import(A, importer, Insert)!=0) {EPETRA_CHK_ERR(-1); } if (importA.FillComplete(A.OperatorDomainMap(), importMap)!=0) {EPETRA_CHK_ERR(-1);} // Finally we are ready to write this strip of the matrix to ostream if (writeRowMatrix(handle, importA)!=0) {EPETRA_CHK_ERR(-1);} } } return(0); }
// ============================================================================ void EpetraExt::XMLWriter:: Write(const std::string& Label, const Epetra_RowMatrix& Matrix) { TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error, "No file has been opened"); int Rows = Matrix.NumGlobalRows(); int Cols = Matrix.NumGlobalRows(); int Nonzeros = Matrix.NumGlobalNonzeros(); if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "<PointMatrix Label=\"" << Label << '"' << " Rows=\"" << Rows << '"' << " Columns=\"" << Cols<< '"' << " Nonzeros=\"" << Nonzeros << '"' << " Type=\"double\" StartingIndex=\"0\">" << std::endl; } int Length = Matrix.MaxNumEntries(); std::vector<int> Indices(Length); std::vector<double> Values(Length); for (int iproc = 0; iproc < Comm_.NumProc(); iproc++) { if (iproc == Comm_.MyPID()) { std::ofstream of(FileName_.c_str(), std::ios::app); of.precision(15); for (int i = 0; i < Matrix.NumMyRows(); ++i) { int NumMyEntries; Matrix.ExtractMyRowCopy(i, Length, NumMyEntries, &Values[0], &Indices[0]); int GRID = Matrix.RowMatrixRowMap().GID(i); for (int j = 0; j < NumMyEntries; ++j) of << GRID << " " << Matrix.RowMatrixColMap().GID(Indices[j]) << " " << std::setiosflags(std::ios::scientific) << Values[j] << std::endl; } of.close(); } Comm_.Barrier(); } if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "</PointMatrix>" << std::endl; of.close(); } }
//============================================================================== int Komplex_LinearProblem::TestMaps (const Epetra_RowMatrix & A0, const Epetra_RowMatrix & A1, const Epetra_MultiVector & Xr, const Epetra_MultiVector & Xi, const Epetra_MultiVector & Br, const Epetra_MultiVector & Bi){ TEUCHOS_TEST_FOR_EXCEPT(!A0.RowMatrixRowMap().SameAs(A1.RowMatrixRowMap())); TEUCHOS_TEST_FOR_EXCEPT(!A0.OperatorDomainMap().SameAs(A1.OperatorDomainMap())); TEUCHOS_TEST_FOR_EXCEPT(!A0.OperatorRangeMap().SameAs(A1.OperatorRangeMap())); TEUCHOS_TEST_FOR_EXCEPT(!A0.OperatorDomainMap().SameAs(Xr.Map())); TEUCHOS_TEST_FOR_EXCEPT(!A0.OperatorRangeMap().SameAs(Br.Map())); TEUCHOS_TEST_FOR_EXCEPT(!A0.OperatorDomainMap().SameAs(Xi.Map())); TEUCHOS_TEST_FOR_EXCEPT(!A0.OperatorRangeMap().SameAs(Bi.Map())); // Test number of vectors also TEUCHOS_TEST_FOR_EXCEPT(Xr.NumVectors()!=Xi.NumVectors()); TEUCHOS_TEST_FOR_EXCEPT(Xr.NumVectors()!=Br.NumVectors()); TEUCHOS_TEST_FOR_EXCEPT(Xr.NumVectors()!=Bi.NumVectors()); return(0); }
int RowMatrixToMatrixMarketFile( const char *filename, const Epetra_RowMatrix & A, const char * matrixName, const char *matrixDescription, bool writeHeader) { long long M = A.NumGlobalRows64(); long long N = A.NumGlobalCols64(); long long nz = A.NumGlobalNonzeros64(); FILE * handle = 0; if (A.RowMatrixRowMap().Comm().MyPID()==0) { // Only PE 0 does this section handle = fopen(filename,"w"); if (!handle) {EPETRA_CHK_ERR(-1);} MM_typecode matcode; mm_initialize_typecode(&matcode); mm_set_matrix(&matcode); mm_set_coordinate(&matcode); mm_set_real(&matcode); if (writeHeader==true) { // Only write header if requested (true by default) if (mm_write_banner(handle, matcode)!=0) {EPETRA_CHK_ERR(-1);} if (matrixName!=0) fprintf(handle, "%% \n%% %s\n", matrixName); if (matrixDescription!=0) fprintf(handle, "%% %s\n%% \n", matrixDescription); if (mm_write_mtx_crd_size(handle, M, N, nz)!=0) {EPETRA_CHK_ERR(-1);} } } if (RowMatrixToHandle(handle, A)!=0) {EPETRA_CHK_ERR(-1);}// Everybody calls this routine if (A.RowMatrixRowMap().Comm().MyPID()==0) // Only PE 0 opened a file if (fclose(handle)!=0) {EPETRA_CHK_ERR(-1);} return(0); }
static int make_my_A(const Epetra_RowMatrix &matrix, int *myA, const Epetra_Comm &comm) { int me = comm.MyPID(); const Epetra_Map &rowmap = matrix.RowMatrixRowMap(); const Epetra_Map &colmap = matrix.RowMatrixColMap(); int myRows = matrix.NumMyRows(); int numRows = matrix.NumGlobalRows(); int numCols = matrix.NumGlobalCols(); int base = rowmap.IndexBase(); int maxRow = matrix.MaxNumEntries(); memset(myA, 0, sizeof(int) * numRows * numCols); int *myIndices = new int [maxRow]; double *tmp = new double [maxRow]; int rowLen = 0; for (int i=0; i< myRows; i++){ int rc = matrix.ExtractMyRowCopy(i, maxRow, rowLen, tmp, myIndices); if (rc){ if (me == 0){ std::cout << "Error in make_my_A" << std::endl; } return 1; } int *row = myA + (numCols * (rowmap.GID(i) - base)); for (int j=0; j < rowLen; j++){ int colGID = colmap.GID(myIndices[j]); row[colGID - base] = me + 1; } } if (maxRow){ delete [] myIndices; delete [] tmp; } return 0; }
int compute_hypergraph_metrics(const Epetra_RowMatrix &matrix, Isorropia::Epetra::CostDescriber &costs, double &myGoalWeight, double &balance, double &cutn, double &cutl) // output { const Epetra_BlockMap &rmap = static_cast<const Epetra_BlockMap &>(matrix.RowMatrixRowMap()); const Epetra_BlockMap &cmap = static_cast<const Epetra_BlockMap &>(matrix.RowMatrixColMap()); return compute_hypergraph_metrics(rmap, cmap, matrix.NumGlobalCols(), costs, myGoalWeight, balance, cutn, cutl); }
int Amesos_Scalapack::NumericFactorization() { if( debug_ == 1 ) std::cout << "Entering `NumericFactorization()'" << std::endl; NumNumericFact_++; iam_ = Comm().MyPID(); Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; NumGlobalElements_ = OriginalMap.NumGlobalElements(); NumGlobalNonzeros_ = RowMatrixA->NumGlobalNonzeros(); RedistributeA(); ConvertToScalapack(); return PerformNumericFactorization( ); }
void BlockCrsMatrix::TSumIntoBlock(double alpha, const Epetra_RowMatrix & BaseMatrix, const int_type Row, const int_type Col) { std::vector<int_type>& RowIndices_ = TRowIndices<int_type>(); std::vector< std::vector<int_type> >& RowStencil_ = TRowStencil<int_type>(); int_type RowOffset = RowIndices_[(std::size_t)Row] * ROffset_; int_type ColOffset = (RowIndices_[(std::size_t)Row] + RowStencil_[(std::size_t)Row][(std::size_t)Col]) * COffset_; // const Epetra_CrsGraph & BaseGraph = BaseMatrix.Graph(); const Epetra_BlockMap & BaseMap = BaseMatrix.RowMatrixRowMap(); const Epetra_BlockMap & BaseColMap = BaseMatrix.RowMatrixColMap(); // This routine copies entries of a BaseMatrix into big BlockCrsMatrix // It performs the following operation on the global IDs row-by-row // this->val[i+rowOffset][j+ColOffset] = BaseMatrix.val[i][j] int MaxIndices = BaseMatrix.MaxNumEntries(); vector<int> Indices_local(MaxIndices); vector<int_type> Indices_global(MaxIndices); vector<double> Values(MaxIndices); int NumIndices; int ierr=0; for (int i=0; i<BaseMap.NumMyElements(); i++) { BaseMatrix.ExtractMyRowCopy( i, MaxIndices, NumIndices, &Values[0], &Indices_local[0] ); // Convert to BlockMatrix Global numbering scheme for( int l = 0; l < NumIndices; ++l ) { Indices_global[l] = ColOffset + (int_type) BaseColMap.GID64(Indices_local[l]); Values[l] *= alpha; } int_type BaseRow = (int_type) BaseMap.GID64(i); ierr = this->SumIntoGlobalValues(BaseRow + RowOffset, NumIndices, &Values[0], &Indices_global[0]); if (ierr != 0) std::cout << "WARNING BlockCrsMatrix::SumIntoBlock SumIntoGlobalValues err = " << ierr << "\n\t Row " << BaseRow + RowOffset << "Col start" << Indices_global[0] << std::endl; } }
int Amesos_Scalapack::ConvertToScalapack(){ // // Convert matrix and vector to the form that Scalapack expects // ScaLAPACK accepts the matrix to be in any 2D block-cyclic form // // Amesos_ScaLAPACK uses one of two 2D data distributions: // a simple 1D non-cyclic data distribution with npcol= 1, or a // full 2D block-cyclic data distribution. // // 2D data distribvution: // Because the Epetra export operation is oriented toward a 1D // data distribution in which each row is entirely stored on // a single process, we create two intermediate matrices: FatIn and // FatOut, both of which have dimension: // NumGlobalElements * nprow by NumGlobalElements // This allows each row of FatOut to be owned by a single process. // The larger dimension does not significantly increase the // storage requirements and allows the export operation to be // efficient. // // 1D data distribution: // We have chosen the simplest 2D block-cyclic form, a 1D blocked (not-cyclic) // data distribution, for the matrix A. // We use the same distribution for the multivectors X and B. However, // except for very large numbers of right hand sides, this places all of X and B // on process 0, making it effectively a serial matrix. // // For now, we simply treat X and B as serial matrices (as viewed from epetra) // though ScaLAPACK treats them as distributed matrices. // if( debug_ == 1 ) std::cout << "Entering `ConvertToScalapack()'" << std::endl; Time_->ResetStartTime(); if ( iam_ < nprow_ * npcol_ ) { if ( TwoD_distribution_ ) { DenseA_.resize( NumOurRows_ * NumOurColumns_ ); for ( int i = 0 ; i < (int)DenseA_.size() ; i++ ) DenseA_[i] = 0 ; assert( lda_ == EPETRA_MAX(1,NumOurRows_) ) ; assert( DescA_[8] == lda_ ) ; int NzThisRow ; int MyRow; double *RowValues; int *ColIndices; int MaxNumEntries = FatOut_->MaxNumEntries(); std::vector<int>ColIndicesV(MaxNumEntries); std::vector<double>RowValuesV(MaxNumEntries); int NumMyElements = FatOut_->NumMyRows() ; for ( MyRow = 0; MyRow < NumMyElements ; MyRow++ ) { EPETRA_CHK_ERR( FatOut_-> ExtractMyRowView( MyRow, NzThisRow, RowValues, ColIndices ) != 0 ) ; // // The following eight lines are just a sanity check on MyRow: // int MyGlobalRow = FatOut_->GRID( MyRow ); assert( MyGlobalRow%npcol_ == mypcol_ ) ; // I should only own rows belonging to my processor column int MyTrueRow = MyGlobalRow/npcol_ ; // This is the original row int UniformRows = ( MyTrueRow / ( nprow_ * nb_ ) ) * nb_ ; int AllExcessRows = MyTrueRow - UniformRows * nprow_ ; int OurExcessRows = AllExcessRows - ( myprow_ * nb_ ) ; if ( MyRow != UniformRows + OurExcessRows ) { std::cout << " iam _ = " << iam_ << " MyGlobalRow = " << MyGlobalRow << " MyTrueRow = " << MyTrueRow << " UniformRows = " << UniformRows << " AllExcessRows = " << AllExcessRows << " OurExcessRows = " << OurExcessRows << " MyRow = " << MyRow << std::endl ; } assert( OurExcessRows >= 0 && OurExcessRows < nb_ ); assert( MyRow == UniformRows + OurExcessRows ) ; for ( int j = 0; j < NzThisRow; j++ ) { assert( FatOut_->RowMatrixColMap().GID( ColIndices[j] ) == FatOut_->GCID( ColIndices[j] ) ); int MyGlobalCol = FatOut_->GCID( ColIndices[j] ); assert( (MyGlobalCol/nb_)%npcol_ == mypcol_ ) ; int UniformCols = ( MyGlobalCol / ( npcol_ * nb_ ) ) * nb_ ; int AllExcessCols = MyGlobalCol - UniformCols * npcol_ ; int OurExcessCols = AllExcessCols - ( mypcol_ * nb_ ) ; assert( OurExcessCols >= 0 && OurExcessCols < nb_ ); int MyCol = UniformCols + OurExcessCols ; DenseA_[ MyCol * lda_ + MyRow ] = RowValues[j] ; } } } else { int NumMyElements = ScaLAPACK1DMatrix_->NumMyRows() ; assert( NumGlobalElements_ ==ScaLAPACK1DMatrix_->NumGlobalRows()); assert( NumGlobalElements_ ==ScaLAPACK1DMatrix_->NumGlobalCols()); DenseA_.resize( NumGlobalElements_ * NumMyElements ) ; for ( int i = 0 ; i < (int)DenseA_.size() ; i++ ) DenseA_[i] = 0 ; int NzThisRow ; int MyRow; double *RowValues; int *ColIndices; int MaxNumEntries = ScaLAPACK1DMatrix_->MaxNumEntries(); assert( DescA_[8] == lda_ ) ; // Double check Lda std::vector<int>ColIndicesV(MaxNumEntries); std::vector<double>RowValuesV(MaxNumEntries); for ( MyRow = 0; MyRow < NumMyElements ; MyRow++ ) { EPETRA_CHK_ERR( ScaLAPACK1DMatrix_-> ExtractMyRowView( MyRow, NzThisRow, RowValues, ColIndices ) != 0 ) ; for ( int j = 0; j < NzThisRow; j++ ) { DenseA_[ ( ScaLAPACK1DMatrix_->RowMatrixColMap().GID( ColIndices[j] ) ) + MyRow * NumGlobalElements_ ] = RowValues[j] ; } } // // Create a map to allow us to redistribute the vectors X and B // Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; assert( NumGlobalElements_ == OriginalMap.NumGlobalElements() ) ; int NumMyElements_ = 0 ; if (iam_==0) NumMyElements_ = NumGlobalElements_; if (VectorMap_) { delete VectorMap_ ; VectorMap_ = 0 ; } VectorMap_ = new Epetra_Map( NumGlobalElements_, NumMyElements_, 0, Comm() ); } } ConTime_ += Time_->ElapsedTime(); return 0; }
int Amesos_Scalapack::RedistributeA( ) { if( debug_ == 1 ) std::cout << "Entering `RedistributeA()'" << std::endl; Time_->ResetStartTime(); Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); EPETRA_CHK_ERR( RowMatrixA == 0 ) ; const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; int NumberOfProcesses = Comm().NumProc() ; // // Compute a uniform distribution as ScaLAPACK would want it // MyFirstElement - The first element which this processor would have // NumExpectedElemetns - The number of elements which this processor would have // int NumRows_ = RowMatrixA->NumGlobalRows() ; int NumColumns_ = RowMatrixA->NumGlobalCols() ; if ( MaxProcesses_ > 0 ) { NumberOfProcesses = EPETRA_MIN( NumberOfProcesses, MaxProcesses_ ) ; } else { int ProcessNumHeuristic = (1+NumRows_/200)*(1+NumRows_/200); NumberOfProcesses = EPETRA_MIN( NumberOfProcesses, ProcessNumHeuristic ); } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:171" << std::endl; // // Create the ScaLAPACK data distribution. // The TwoD data distribution is created in a completely different // manner and is not transposed (whereas the SaLAPACK 1D data // distribution was transposed) // if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:163" << std::endl; Comm().Barrier(); if ( TwoD_distribution_ ) { if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:166" << std::endl; Comm().Barrier(); npcol_ = EPETRA_MIN( NumberOfProcesses, EPETRA_MAX ( 2, (int) sqrt( NumberOfProcesses * 0.5 ) ) ) ; nprow_ = NumberOfProcesses / npcol_ ; // // Create the map for FatA - our first intermediate matrix // int NumMyElements = RowMatrixA->RowMatrixRowMap().NumMyElements() ; std::vector<int> MyGlobalElements( NumMyElements ); RowMatrixA->RowMatrixRowMap().MyGlobalElements( &MyGlobalElements[0] ) ; int NumMyColumns = RowMatrixA->RowMatrixColMap().NumMyElements() ; std::vector<int> MyGlobalColumns( NumMyColumns ); RowMatrixA->RowMatrixColMap().MyGlobalElements( &MyGlobalColumns[0] ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:194" << std::endl; std::vector<int> MyFatElements( NumMyElements * npcol_ ); for( int LocalRow=0; LocalRow<NumMyElements; LocalRow++ ) { for (int i = 0 ; i < npcol_; i++ ){ MyFatElements[LocalRow*npcol_+i] = MyGlobalElements[LocalRow]*npcol_+i; } } Epetra_Map FatInMap( npcol_*NumRows_, NumMyElements*npcol_, &MyFatElements[0], 0, Comm() ); // // Create FatIn, our first intermediate matrix // Epetra_CrsMatrix FatIn( Copy, FatInMap, 0 ); std::vector<std::vector<int> > FatColumnIndices(npcol_,std::vector<int>(1)); std::vector<std::vector<double> > FatMatrixValues(npcol_,std::vector<double>(1)); std::vector<int> FatRowPtrs(npcol_); // A FatRowPtrs[i] = the number // of entries in local row LocalRow*npcol_ + i if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:219" << std::endl; // mypcol_ = iam_%npcol_; myprow_ = (iam_/npcol_)%nprow_; if ( iam_ >= nprow_ * npcol_ ) { myprow_ = nprow_; mypcol_ = npcol_; } // Each row is split into npcol_ rows, with each of the // new rows containing only those elements belonging to // its process column (in the ScaLAPACK 2D process grid) // int MaxNumIndices = RowMatrixA->MaxNumEntries(); int NumIndices; std::vector<int> ColumnIndices(MaxNumIndices); std::vector<double> MatrixValues(MaxNumIndices); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:232 NumMyElements = " << NumMyElements << std::endl; nb_ = grid_nb_; for( int LocalRow=0; LocalRow<NumMyElements; ++LocalRow ) { RowMatrixA->ExtractMyRowCopy( LocalRow, MaxNumIndices, NumIndices, &MatrixValues[0], &ColumnIndices[0] ); for (int i=0; i<npcol_; i++ ) FatRowPtrs[i] = 0 ; // // Deal the individual matrix entries out to the row owned by // the process to which this matrix entry will belong. // for( int i=0 ; i<NumIndices ; ++i ) { int GlobalCol = MyGlobalColumns[ ColumnIndices[i] ]; int pcol_i = pcolnum( GlobalCol, nb_, npcol_ ) ; if ( FatRowPtrs[ pcol_i ]+1 >= FatColumnIndices[ pcol_i ].size() ) { FatColumnIndices[ pcol_i ]. resize( 2 * FatRowPtrs[ pcol_i ]+1 ); FatMatrixValues[ pcol_i ]. resize( 2 * FatRowPtrs[ pcol_i ]+1 ); } FatColumnIndices[pcol_i][FatRowPtrs[pcol_i]] = GlobalCol ; FatMatrixValues[pcol_i][FatRowPtrs[pcol_i]] = MatrixValues[i]; FatRowPtrs[ pcol_i ]++; } // // Insert each of the npcol_ rows individually // for ( int pcol_i = 0 ; pcol_i < npcol_ ; pcol_i++ ) { FatIn.InsertGlobalValues( MyGlobalElements[LocalRow]*npcol_ + pcol_i, FatRowPtrs[ pcol_i ], &FatMatrixValues[ pcol_i ][0], &FatColumnIndices[ pcol_i ][0] ); } } FatIn.FillComplete( false ); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:260" << std::endl; if ( debug_ == 1) std::cout << "Amesos_Scalapack.cpp:265B" << " iam_ = " << iam_ << " nb_ = " << nb_ << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << std::endl; // // Compute the map for our second intermediate matrix, FatOut // // Compute directly int UniformRows = ( NumRows_ / ( nprow_ * nb_ ) ) * nb_ ; int AllExcessRows = NumRows_ - UniformRows * nprow_ ; int OurExcessRows = EPETRA_MIN( nb_, AllExcessRows - ( myprow_ * nb_ ) ) ; OurExcessRows = EPETRA_MAX( 0, OurExcessRows ); NumOurRows_ = UniformRows + OurExcessRows ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:277" << std::endl; int UniformColumns = ( NumColumns_ / ( npcol_ * nb_ ) ) * nb_ ; int AllExcessColumns = NumColumns_ - UniformColumns * npcol_ ; int OurExcessColumns = EPETRA_MIN( nb_, AllExcessColumns - ( mypcol_ * nb_ ) ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:281" << std::endl; OurExcessColumns = EPETRA_MAX( 0, OurExcessColumns ); NumOurColumns_ = UniformColumns + OurExcessColumns ; if ( iam_ >= nprow_ * npcol_ ) { UniformRows = 0; NumOurRows_ = 0; NumOurColumns_ = 0; } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:295" << std::endl; #if 0 // Compute using ScaLAPACK's numroc routine, assert agreement int izero = 0; // All matrices start at process 0 int NumRocSays = numroc_( &NumRows_, &nb_, &myprow_, &izero, &nprow_ ); assert( NumOurRows_ == NumRocSays ); #endif // // Compute the rows which this process row owns in the ScaLAPACK 2D // process grid. // std::vector<int> AllOurRows(NumOurRows_); int RowIndex = 0 ; int BlockRow = 0 ; for ( ; BlockRow < UniformRows / nb_ ; BlockRow++ ) { for ( int RowOffset = 0; RowOffset < nb_ ; RowOffset++ ) { AllOurRows[RowIndex++] = BlockRow*nb_*nprow_ + myprow_*nb_ + RowOffset ; } } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:315" << std::endl; assert ( BlockRow == UniformRows / nb_ ) ; for ( int RowOffset = 0; RowOffset < OurExcessRows ; RowOffset++ ) { AllOurRows[RowIndex++] = BlockRow*nb_*nprow_ + myprow_*nb_ + RowOffset ; } assert( RowIndex == NumOurRows_ ); // // Distribute those rows amongst all the processes in that process row // This is an artificial distribution with the following properties: // 1) It is a 1D data distribution (each row belogs entirely to // a single process // 2) All data which will eventually belong to a given process row, // is entirely contained within the processes in that row. // Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:312" << std::endl; // // Compute MyRows directly // std::vector<int>MyRows(NumOurRows_); RowIndex = 0 ; BlockRow = 0 ; for ( ; BlockRow < UniformRows / nb_ ; BlockRow++ ) { for ( int RowOffset = 0; RowOffset < nb_ ; RowOffset++ ) { MyRows[RowIndex++] = BlockRow*nb_*nprow_*npcol_ + myprow_*nb_*npcol_ + RowOffset*npcol_ + mypcol_ ; } } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:326" << std::endl; assert ( BlockRow == UniformRows / nb_ ) ; for ( int RowOffset = 0; RowOffset < OurExcessRows ; RowOffset++ ) { MyRows[RowIndex++] = BlockRow*nb_*nprow_*npcol_ + myprow_*nb_*npcol_ + RowOffset*npcol_ + mypcol_ ; } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:334" << std::endl; Comm().Barrier(); for (int i=0; i < NumOurRows_; i++ ) { assert( MyRows[i] == AllOurRows[i]*npcol_+mypcol_ ); } Comm().Barrier(); if ( debug_ == 1) std::cout << "Amesos_Scalapack.cpp:340" << " iam_ = " << iam_ << " myprow_ = " << myprow_ << " mypcol_ = " << mypcol_ << " NumRows_ = " << NumRows_ << " NumOurRows_ = " << NumOurRows_ << std::endl; Comm().Barrier(); Epetra_Map FatOutMap( npcol_*NumRows_, NumOurRows_, &MyRows[0], 0, Comm() ); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:344" << std::endl; Comm().Barrier(); if ( FatOut_ ) delete FatOut_ ; FatOut_ = new Epetra_CrsMatrix( Copy, FatOutMap, 0 ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:348" << std::endl; Epetra_Export ExportToFatOut( FatInMap, FatOutMap ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:360" << std::endl; FatOut_->Export( FatIn, ExportToFatOut, Add ); FatOut_->FillComplete( false ); // // Create a map to allow us to redistribute the vectors X and B // Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; assert( NumGlobalElements_ == OriginalMap.NumGlobalElements() ) ; int NumMyVecElements = 0 ; if ( mypcol_ == 0 ) { NumMyVecElements = NumOurRows_; } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:385" << std::endl; if (VectorMap_) { delete VectorMap_ ; VectorMap_ = 0 ; } VectorMap_ = new Epetra_Map( NumGlobalElements_, NumMyVecElements, &AllOurRows[0], 0, Comm() ); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << " Amesos_Scalapack.cpp:393 debug_ = " << debug_ << std::endl; } else { nprow_ = 1 ; npcol_ = NumberOfProcesses / nprow_ ; assert ( nprow_ * npcol_ == NumberOfProcesses ) ; m_per_p_ = ( NumRows_ + NumberOfProcesses - 1 ) / NumberOfProcesses ; int MyFirstElement = EPETRA_MIN( iam_ * m_per_p_, NumRows_ ) ; int MyFirstNonElement = EPETRA_MIN( (iam_+1) * m_per_p_, NumRows_ ) ; int NumExpectedElements = MyFirstNonElement - MyFirstElement ; assert( NumRows_ == RowMatrixA->NumGlobalRows() ) ; if ( ScaLAPACK1DMap_ ) delete( ScaLAPACK1DMap_ ) ; ScaLAPACK1DMap_ = new Epetra_Map( NumRows_, NumExpectedElements, 0, Comm() ); if ( ScaLAPACK1DMatrix_ ) delete( ScaLAPACK1DMatrix_ ) ; ScaLAPACK1DMatrix_ = new Epetra_CrsMatrix(Copy, *ScaLAPACK1DMap_, 0); Epetra_Export ExportToScaLAPACK1D_( OriginalMap, *ScaLAPACK1DMap_); ScaLAPACK1DMatrix_->Export( *RowMatrixA, ExportToScaLAPACK1D_, Add ); ScaLAPACK1DMatrix_->FillComplete( false ) ; } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << " Amesos_Scalapack.cpp:417 debug_ = " << debug_ << std::endl; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:402" << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << std::endl ; int info; const int zero = 0 ; if ( ictxt_ == -1313 ) { ictxt_ = 0 ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:408" << std::endl; SL_INIT_F77(&ictxt_, &nprow_, &npcol_) ; } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:410A" << std::endl; int nprow; int npcol; int myrow; int mycol; BLACS_GRIDINFO_F77(&ictxt_, &nprow, &npcol, &myrow, &mycol) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "iam_ = " << iam_ << " Amesos_Scalapack.cpp:410" << std::endl; if ( iam_ < nprow_ * npcol_ ) { assert( nprow == nprow_ ) ; if ( npcol != npcol_ ) std::cout << "Amesos_Scalapack.cpp:430 npcol = " << npcol << " npcol_ = " << npcol_ << std::endl ; assert( npcol == npcol_ ) ; if ( TwoD_distribution_ ) { assert( myrow == myprow_ ) ; assert( mycol == mypcol_ ) ; lda_ = EPETRA_MAX(1,NumOurRows_) ; } else { assert( myrow == 0 ) ; assert( mycol == iam_ ) ; nb_ = m_per_p_; lda_ = EPETRA_MAX(1,NumGlobalElements_); } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp: " << __LINE__ << " TwoD_distribution_ = " << TwoD_distribution_ << " NumGlobalElements_ = " << NumGlobalElements_ << " debug_ = " << debug_ << " nb_ = " << nb_ << " lda_ = " << lda_ << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << " myprow_ = " << myprow_ << " mypcol_ = " << mypcol_ << " iam_ = " << iam_ << std::endl ; AMESOS_PRINT( myprow_ ); DESCINIT_F77(DescA_, &NumGlobalElements_, &NumGlobalElements_, &nb_, &nb_, &zero, &zero, &ictxt_, &lda_, &info) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:441" << std::endl; assert( info == 0 ) ; } else { DescA_[0] = -13; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:458 nprow = " << nprow << std::endl; assert( nprow == -1 ) ; } if ( debug_ == 1) std::cout << "Amesos_Scalapack.cpp:446" << std::endl; MatTime_ += Time_->ElapsedTime(); return 0; }
bool BroydenOperator::computeSparseBroydenUpdate() { if( isValidBroyden ) return true; else if( isValidStep && isValidYield ) { // Do the Broyden update to our matrix int ierr = crsMatrix->Multiply( false, stepVec->getEpetraVector(), workVec->getEpetraVector() ); if( ierr ) { cout << "ERROR: NOX::Epetra::BroydenOperator::computeSparseBroydenUpdate(...) " << "- crsMatrix->Multiply() failed!!!" << endl; throw "NOX Error: Broyden Update Failed"; } int numEntries = crsMatrix->NumMyCols(); int * indices ; double * values ; for( int row = 0; row < crsMatrix->NumMyRows(); ++row ) { ierr = crsMatrix->ExtractMyRowView( row, numEntries, values, indices ); if( ierr ) { cout << "ERROR (" << ierr << ") : NOX::Epetra::BroydenOperator::computeSparseBroydenUpdate() " << "- crsMatrix->ExtractGlobalRowView(...) failed for row --> " << row << endl; throw "NOX Error: Broyden Update Failed"; } double diffVal = yieldVec->getEpetraVector()[row] - workVec->getEpetraVector()[row]; double rowStepInnerProduct = 0.0; if( entriesRemoved[row] ) { list<int>::iterator iter = retainedEntries[row].begin() , iter_end = retainedEntries[row].end() ; for( ; iter_end != iter; ++iter ) rowStepInnerProduct += stepVec->getEpetraVector()[indices[*iter]] * stepVec->getEpetraVector()[indices[*iter]]; for( iter = retainedEntries[row].begin(); iter_end != iter; ++iter ) values[*iter] += diffVal * stepVec->getEpetraVector()[indices[*iter]] / rowStepInnerProduct; } else { for( int col = 0; col < numEntries; ++col ) rowStepInnerProduct += stepVec->getEpetraVector()[indices[col]] * stepVec->getEpetraVector()[indices[col]]; for( int col = 0; col < numEntries; ++col ) (*values++) += diffVal * stepVec->getEpetraVector()[(*indices++)] / rowStepInnerProduct; } } // Our %Broyden matrix has been updated and is now ready to use as a // preconditioning matrix or as the Jacobian. if( verbose ) cout << "\t... BroydenOperator::computeSparseBroydenUpdate()..." << endl; } else { cout << "Warning: NOX::Epetra::BroydenOperator::computeSparseBroydenUpdate(...) " << "- either the step vector or the yield vector or both is not valid." << endl; cout << "Leaving existing matrix unchanged." << endl; } // Use EpetraExt to dump linear system if debuggging #ifdef HAVE_NOX_DEBUG #ifdef HAVE_NOX_EPETRAEXT static int broydenOutputCount; Teuchos::ParameterList & broydenParamas = nlParams.sublist("Direction").sublist("Newton").sublist("Broyden Op"); if( broydenParamas.get("Write Broyden Info", false) ) { std::ostringstream outputNumber; outputNumber << broydenOutputCount++ ; std::string prefixName = broydenParamas.get("Write Broyden Info File Prefix", "BroydenOp"); std::string postfixName = outputNumber.str(); postfixName += ".mm"; std::string mapFileName = prefixName + "_Map_" + postfixName; std::string matFileName = prefixName + "_Matrix_" + postfixName; std::string dxFileName = prefixName + "_dX_" + postfixName; std::string dfFileName = prefixName + "_dF_" + postfixName; Epetra_RowMatrix * printMatrix = NULL; printMatrix = dynamic_cast<Epetra_RowMatrix*>(crsMatrix.get()); if( NULL == printMatrix ) { cout << "Error: NOX::Epetra::BroydenOperator::computeSparseBroydenUpdate() - " << "Could not get a valid crsMatrix!\n" << "Please set the \"Write Linear System\" parameter to false." << endl; throw "NOX Error"; } EpetraExt::BlockMapToMatrixMarketFile(mapFileName.c_str(), printMatrix->RowMatrixRowMap()); EpetraExt::RowMatrixToMatrixMarketFile(matFileName.c_str(), *printMatrix, "test matrix", "Broyden Matrix XXX"); EpetraExt::MultiVectorToMatrixMarketFile(dxFileName.c_str(), stepVec->getEpetraVector()); EpetraExt::MultiVectorToMatrixMarketFile(dfFileName.c_str(), yieldVec->getEpetraVector()); } #endif #endif return true; }
//============================================================================= int Amesos_Dscpack::PerformSymbolicFactorization() { ResetTimer(0); ResetTimer(1); MyPID_ = Comm().MyPID(); NumProcs_ = Comm().NumProc(); Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix(); if (RowMatrixA == 0) AMESOS_CHK_ERR(-1); const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ; const Epetra_MpiComm& comm1 = dynamic_cast<const Epetra_MpiComm &> (Comm()); int numrows = RowMatrixA->NumGlobalRows(); int numentries = RowMatrixA->NumGlobalNonzeros(); Teuchos::RCP<Epetra_CrsGraph> Graph; Epetra_CrsMatrix* CastCrsMatrixA = dynamic_cast<Epetra_CrsMatrix*>(RowMatrixA); if (CastCrsMatrixA) { Graph = Teuchos::rcp(const_cast<Epetra_CrsGraph*>(&(CastCrsMatrixA->Graph())), false); } else { int MaxNumEntries = RowMatrixA->MaxNumEntries(); Graph = Teuchos::rcp(new Epetra_CrsGraph(Copy, OriginalMap, MaxNumEntries)); std::vector<int> Indices(MaxNumEntries); std::vector<double> Values(MaxNumEntries); for (int i = 0 ; i < RowMatrixA->NumMyRows() ; ++i) { int NumEntries; RowMatrixA->ExtractMyRowCopy(i, MaxNumEntries, NumEntries, &Values[0], &Indices[0]); for (int j = 0 ; j < NumEntries ; ++j) Indices[j] = RowMatrixA->RowMatrixColMap().GID(Indices[j]); int GlobalRow = RowMatrixA->RowMatrixRowMap().GID(i); Graph->InsertGlobalIndices(GlobalRow, NumEntries, &Indices[0]); } Graph->FillComplete(); } // // Create a replicated map and graph // std::vector<int> AllIDs( numrows ) ; for ( int i = 0; i < numrows ; i++ ) AllIDs[i] = i ; Epetra_Map ReplicatedMap( -1, numrows, &AllIDs[0], 0, Comm()); Epetra_Import ReplicatedImporter(ReplicatedMap, OriginalMap); Epetra_CrsGraph ReplicatedGraph( Copy, ReplicatedMap, 0 ); AMESOS_CHK_ERR(ReplicatedGraph.Import(*Graph, ReplicatedImporter, Insert)); AMESOS_CHK_ERR(ReplicatedGraph.FillComplete()); // // Convert the matrix to Ap, Ai // std::vector <int> Replicates(numrows); std::vector <int> Ap(numrows + 1); std::vector <int> Ai(EPETRA_MAX(numrows, numentries)); for( int i = 0 ; i < numrows; i++ ) Replicates[i] = 1; int NumEntriesPerRow ; int *ColIndices = 0 ; int Ai_index = 0 ; for ( int MyRow = 0; MyRow <numrows; MyRow++ ) { AMESOS_CHK_ERR( ReplicatedGraph.ExtractMyRowView( MyRow, NumEntriesPerRow, ColIndices ) ); Ap[MyRow] = Ai_index ; for ( int j = 0; j < NumEntriesPerRow; j++ ) { Ai[Ai_index] = ColIndices[j] ; Ai_index++; } } assert( Ai_index == numentries ) ; Ap[ numrows ] = Ai_index ; MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0); ResetTimer(0); // // Call Dscpack Symbolic Factorization // int OrderCode = 2; std::vector<double> MyANonZ; NumLocalNonz = 0 ; GlobalStructNewColNum = 0 ; GlobalStructNewNum = 0 ; GlobalStructOwner = 0 ; LocalStructOldNum = 0 ; NumGlobalCols = 0 ; // MS // Have to define the maximum number of processes to be used // MS // This is only a suggestion as Dscpack uses a number of processes that is a power of 2 int NumGlobalNonzeros = GetProblem()->GetMatrix()->NumGlobalNonzeros(); int NumRows = GetProblem()->GetMatrix()->NumGlobalRows(); // optimal value for MaxProcs == -1 int OptNumProcs1 = 1+EPETRA_MAX( NumRows/10000, NumGlobalNonzeros/1000000 ); OptNumProcs1 = EPETRA_MIN(NumProcs_,OptNumProcs1 ); // optimal value for MaxProcs == -2 int OptNumProcs2 = (int)sqrt(1.0 * NumProcs_); if( OptNumProcs2 < 1 ) OptNumProcs2 = 1; // fix the value of MaxProcs switch (MaxProcs_) { case -1: MaxProcs_ = EPETRA_MIN(OptNumProcs1, NumProcs_); break; case -2: MaxProcs_ = EPETRA_MIN(OptNumProcs2, NumProcs_); break; case -3: MaxProcs_ = NumProcs_; break; } #if 0 if (MyDscRank>=0 && A_and_LU_built) { DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject); } #endif // if ( ! A_and_LU_built ) { // DSC_End( PrivateDscpackData_->MyDSCObject ) ; // PrivateDscpackData_->MyDSCObject = DSC_Begin() ; // } // MS // here I continue with the old code... OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); DscNumProcs = 1 ; int DscMax = DSC_Analyze( numrows, &Ap[0], &Ai[0], &Replicates[0] ); while ( DscNumProcs * 2 <=EPETRA_MIN( MaxProcs_, DscMax ) ) DscNumProcs *= 2 ; MyDscRank = -1; DSC_Open0( PrivateDscpackData_->MyDSCObject_, DscNumProcs, &MyDscRank, comm1.Comm()) ; NumLocalCols = 0 ; // This is for those processes not in the Dsc grid if ( MyDscRank >= 0 ) { assert( MyPID_ == MyDscRank ) ; AMESOS_CHK_ERR( DSC_Order ( PrivateDscpackData_->MyDSCObject_, OrderCode, numrows, &Ap[0], &Ai[0], &Replicates[0], &NumGlobalCols, &NumLocalStructs, &NumLocalCols, &NumLocalNonz, &GlobalStructNewColNum, &GlobalStructNewNum, &GlobalStructOwner, &LocalStructOldNum ) ) ; assert( NumGlobalCols == numrows ) ; assert( NumLocalCols == NumLocalStructs ) ; } if ( MyDscRank >= 0 ) { int MaxSingleBlock; const int Limit = 5000000 ; // Memory Limit set to 5 Terabytes AMESOS_CHK_ERR( DSC_SFactor ( PrivateDscpackData_->MyDSCObject_, &TotalMemory_, &MaxSingleBlock, Limit, DSC_LBLAS3, DSC_DBLAS2 ) ) ; } // A_and_LU_built = true; // If you uncomment this, TestOptions fails SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_, 0); return(0); }
int CopyRowMatrix(mxArray* matlabA, const Epetra_RowMatrix& A) { int valueCount = 0; //int* valueCount = &temp; Epetra_Map map = A.RowMatrixRowMap(); const Epetra_Comm & comm = map.Comm(); int numProc = comm.NumProc(); if (numProc==1) DoCopyRowMatrix(matlabA, valueCount, A); else { int numRows = map.NumMyElements(); //cout << "creating allGidsMap\n"; Epetra_Map allGidsMap(-1, numRows, 0,comm); //cout << "done creating allGidsMap\n"; Epetra_IntVector allGids(allGidsMap); for (int i=0; i<numRows; i++) allGids[i] = map.GID(i); // Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A. int numChunks = numProc; int stripSize = allGids.GlobalLength()/numChunks; int remainder = allGids.GlobalLength()%numChunks; int curStart = 0; int curStripSize = 0; Epetra_IntSerialDenseVector importGidList; int numImportGids = 0; if (comm.MyPID()==0) importGidList.Size(stripSize+1); // Set size of vector to max needed for (int i=0; i<numChunks; i++) { if (comm.MyPID()==0) { // Only PE 0 does this part curStripSize = stripSize; if (i<remainder) curStripSize++; // handle leftovers for (int j=0; j<curStripSize; j++) importGidList[j] = j + curStart; curStart += curStripSize; } // The following import map will be non-trivial only on PE 0. //cout << "creating importGidMap\n"; Epetra_Map importGidMap(-1, curStripSize, importGidList.Values(), 0, comm); //cout << "done creating importGidMap\n"; Epetra_Import gidImporter(importGidMap, allGidsMap); Epetra_IntVector importGids(importGidMap); if (importGids.Import(allGids, gidImporter, Insert)) return(-1); // importGids now has a list of GIDs for the current strip of matrix rows. // Use these values to build another importer that will get rows of the matrix. // The following import map will be non-trivial only on PE 0. //cout << "creating importMap\n"; //cout << "A.RowMatrixRowMap().MinAllGID: " << A.RowMatrixRowMap().MinAllGID() << "\n"; Epetra_Map importMap(-1, importGids.MyLength(), importGids.Values(), A.RowMatrixRowMap().MinAllGID(), comm); //cout << "done creating importMap\n"; Epetra_Import importer(importMap, map); Epetra_CrsMatrix importA(Copy, importMap, 0); if (importA.Import(A, importer, Insert)) return(-1); if (importA.FillComplete()) return(-1); // Finally we are ready to write this strip of the matrix to ostream if (DoCopyRowMatrix(matlabA, valueCount, importA)) return(-1); } } if (A.RowMatrixRowMap().Comm().MyPID() == 0) { // set max cap int* matlabAcolumnIndicesPtr = mxGetJc(matlabA); matlabAcolumnIndicesPtr[A.NumGlobalRows()] = valueCount; } return(0); }
int Ifpack_Analyze(const Epetra_RowMatrix& A, const bool Cheap, const int NumPDEEqns) { int NumMyRows = A.NumMyRows(); long long NumGlobalRows = A.NumGlobalRows64(); long long NumGlobalCols = A.NumGlobalCols64(); long long MyBandwidth = 0, GlobalBandwidth; long long MyLowerNonzeros = 0, MyUpperNonzeros = 0; long long GlobalLowerNonzeros, GlobalUpperNonzeros; long long MyDiagonallyDominant = 0, GlobalDiagonallyDominant; long long MyWeaklyDiagonallyDominant = 0, GlobalWeaklyDiagonallyDominant; double MyMin, MyAvg, MyMax; double GlobalMin, GlobalAvg, GlobalMax; long long GlobalStorage; bool verbose = (A.Comm().MyPID() == 0); GlobalStorage = sizeof(int*) * NumGlobalRows + sizeof(int) * A.NumGlobalNonzeros64() + sizeof(double) * A.NumGlobalNonzeros64(); if (verbose) { print(); Ifpack_PrintLine(); print<const char*>("Label", A.Label()); print<long long>("Global rows", NumGlobalRows); print<long long>("Global columns", NumGlobalCols); print<long long>("Stored nonzeros", A.NumGlobalNonzeros64()); print<long long>("Nonzeros / row", A.NumGlobalNonzeros64() / NumGlobalRows); print<double>("Estimated storage (Mbytes)", 1.0e-6 * GlobalStorage); } long long NumMyActualNonzeros = 0, NumGlobalActualNonzeros; long long NumMyEmptyRows = 0, NumGlobalEmptyRows; long long NumMyDirichletRows = 0, NumGlobalDirichletRows; std::vector<int> colInd(A.MaxNumEntries()); std::vector<double> colVal(A.MaxNumEntries()); Epetra_Vector Diag(A.RowMatrixRowMap()); Epetra_Vector RowSum(A.RowMatrixRowMap()); Diag.PutScalar(0.0); RowSum.PutScalar(0.0); for (int i = 0 ; i < NumMyRows ; ++i) { long long GRID = A.RowMatrixRowMap().GID64(i); int Nnz; IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz, &colVal[0],&colInd[0])); if (Nnz == 0) NumMyEmptyRows++; if (Nnz == 1) NumMyDirichletRows++; for (int j = 0 ; j < Nnz ; ++j) { double v = colVal[j]; if (v < 0) v = -v; if (colVal[j] != 0.0) NumMyActualNonzeros++; long long GCID = A.RowMatrixColMap().GID64(colInd[j]); if (GCID != GRID) RowSum[i] += v; else Diag[i] = v; if (GCID < GRID) MyLowerNonzeros++; else if (GCID > GRID) MyUpperNonzeros++; long long b = GCID - GRID; if (b < 0) b = -b; if (b > MyBandwidth) MyBandwidth = b; } if (Diag[i] > RowSum[i]) MyDiagonallyDominant++; if (Diag[i] >= RowSum[i]) MyWeaklyDiagonallyDominant++; RowSum[i] += Diag[i]; } // ======================== // // summing up global values // // ======================== // A.Comm().SumAll(&MyDiagonallyDominant,&GlobalDiagonallyDominant,1); A.Comm().SumAll(&MyWeaklyDiagonallyDominant,&GlobalWeaklyDiagonallyDominant,1); A.Comm().SumAll(&NumMyActualNonzeros, &NumGlobalActualNonzeros, 1); A.Comm().SumAll(&NumMyEmptyRows, &NumGlobalEmptyRows, 1); A.Comm().SumAll(&NumMyDirichletRows, &NumGlobalDirichletRows, 1); A.Comm().SumAll(&MyBandwidth, &GlobalBandwidth, 1); A.Comm().SumAll(&MyLowerNonzeros, &GlobalLowerNonzeros, 1); A.Comm().SumAll(&MyUpperNonzeros, &GlobalUpperNonzeros, 1); A.Comm().SumAll(&MyDiagonallyDominant, &GlobalDiagonallyDominant, 1); A.Comm().SumAll(&MyWeaklyDiagonallyDominant, &GlobalWeaklyDiagonallyDominant, 1); double NormOne = A.NormOne(); double NormInf = A.NormInf(); double NormF = Ifpack_FrobeniusNorm(A); if (verbose) { print(); print<long long>("Actual nonzeros", NumGlobalActualNonzeros); print<long long>("Nonzeros in strict lower part", GlobalLowerNonzeros); print<long long>("Nonzeros in strict upper part", GlobalUpperNonzeros); print(); print<long long>("Empty rows", NumGlobalEmptyRows, 100.0 * NumGlobalEmptyRows / NumGlobalRows); print<long long>("Dirichlet rows", NumGlobalDirichletRows, 100.0 * NumGlobalDirichletRows / NumGlobalRows); print<long long>("Diagonally dominant rows", GlobalDiagonallyDominant, 100.0 * GlobalDiagonallyDominant / NumGlobalRows); print<long long>("Weakly diag. dominant rows", GlobalWeaklyDiagonallyDominant, 100.0 * GlobalWeaklyDiagonallyDominant / NumGlobalRows); print(); print<long long>("Maximum bandwidth", GlobalBandwidth); print(); print("", "one-norm", "inf-norm", "Frobenius", false); print("", "========", "========", "=========", false); print(); print<double>("A", NormOne, NormInf, NormF); } if (Cheap == false) { // create A + A^T and A - A^T Epetra_FECrsMatrix AplusAT(Copy, A.RowMatrixRowMap(), 0); Epetra_FECrsMatrix AminusAT(Copy, A.RowMatrixRowMap(), 0); #ifndef EPETRA_NO_32BIT_GLOBAL_INDICES if(A.RowMatrixRowMap().GlobalIndicesInt()) { for (int i = 0 ; i < NumMyRows ; ++i) { int GRID = A.RowMatrixRowMap().GID(i); assert (GRID != -1); int Nnz; IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz, &colVal[0],&colInd[0])); for (int j = 0 ; j < Nnz ; ++j) { int GCID = A.RowMatrixColMap().GID(colInd[j]); assert (GCID != -1); double plus_val = colVal[j]; double minus_val = -colVal[j]; if (AplusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) { IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val)); } if (AplusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&plus_val) != 0) { IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GCID,1,&GRID,&plus_val)); } if (AminusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) { IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val)); } if (AminusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&minus_val) != 0) { IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GCID,1,&GRID,&minus_val)); } } } } else #endif #ifndef EPETRA_NO_64BIT_GLOBAL_INDICES if(A.RowMatrixRowMap().GlobalIndicesLongLong()) { for (int i = 0 ; i < NumMyRows ; ++i) { long long GRID = A.RowMatrixRowMap().GID64(i); assert (GRID != -1); int Nnz; IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz, &colVal[0],&colInd[0])); for (int j = 0 ; j < Nnz ; ++j) { long long GCID = A.RowMatrixColMap().GID64(colInd[j]); assert (GCID != -1); double plus_val = colVal[j]; double minus_val = -colVal[j]; if (AplusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) { IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val)); } if (AplusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&plus_val) != 0) { IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GCID,1,&GRID,&plus_val)); } if (AminusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) { IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val)); } if (AminusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&minus_val) != 0) { IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GCID,1,&GRID,&minus_val)); } } } } else #endif throw "Ifpack_Analyze: GlobalIndices type unknown"; AplusAT.FillComplete(); AminusAT.FillComplete(); AplusAT.Scale(0.5); AminusAT.Scale(0.5); NormOne = AplusAT.NormOne(); NormInf = AplusAT.NormInf(); NormF = Ifpack_FrobeniusNorm(AplusAT); if (verbose) { print<double>("A + A^T", NormOne, NormInf, NormF); } NormOne = AminusAT.NormOne(); NormInf = AminusAT.NormInf(); NormF = Ifpack_FrobeniusNorm(AminusAT); if (verbose) { print<double>("A - A^T", NormOne, NormInf, NormF); } } if (verbose) { print(); print<const char*>("", "min", "avg", "max", false); print<const char*>("", "===", "===", "===", false); } MyMax = -DBL_MAX; MyMin = DBL_MAX; MyAvg = 0.0; for (int i = 0 ; i < NumMyRows ; ++i) { int Nnz; IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz, &colVal[0],&colInd[0])); for (int j = 0 ; j < Nnz ; ++j) { MyAvg += colVal[j]; if (colVal[j] > MyMax) MyMax = colVal[j]; if (colVal[j] < MyMin) MyMin = colVal[j]; } } A.Comm().MaxAll(&MyMax, &GlobalMax, 1); A.Comm().MinAll(&MyMin, &GlobalMin, 1); A.Comm().SumAll(&MyAvg, &GlobalAvg, 1); GlobalAvg /= A.NumGlobalNonzeros64(); if (verbose) { print(); print<double>(" A(i,j)", GlobalMin, GlobalAvg, GlobalMax); } MyMax = 0.0; MyMin = DBL_MAX; MyAvg = 0.0; for (int i = 0 ; i < NumMyRows ; ++i) { int Nnz; IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz, &colVal[0],&colInd[0])); for (int j = 0 ; j < Nnz ; ++j) { double v = colVal[j]; if (v < 0) v = -v; MyAvg += v; if (colVal[j] > MyMax) MyMax = v; if (colVal[j] < MyMin) MyMin = v; } } A.Comm().MaxAll(&MyMax, &GlobalMax, 1); A.Comm().MinAll(&MyMin, &GlobalMin, 1); A.Comm().SumAll(&MyAvg, &GlobalAvg, 1); GlobalAvg /= A.NumGlobalNonzeros64(); if (verbose) { print<double>("|A(i,j)|", GlobalMin, GlobalAvg, GlobalMax); } // ================= // // diagonal elements // // ================= // Diag.MinValue(&GlobalMin); Diag.MaxValue(&GlobalMax); Diag.MeanValue(&GlobalAvg); if (verbose) { print(); print<double>(" A(k,k)", GlobalMin, GlobalAvg, GlobalMax); } Diag.Abs(Diag); Diag.MinValue(&GlobalMin); Diag.MaxValue(&GlobalMax); Diag.MeanValue(&GlobalAvg); if (verbose) { print<double>("|A(k,k)|", GlobalMin, GlobalAvg, GlobalMax); } // ============================================== // // cycle over all equations for diagonal elements // // ============================================== // if (NumPDEEqns > 1 ) { if (verbose) print(); for (int ie = 0 ; ie < NumPDEEqns ; ie++) { MyMin = DBL_MAX; MyMax = -DBL_MAX; MyAvg = 0.0; for (int i = ie ; i < Diag.MyLength() ; i += NumPDEEqns) { double d = Diag[i]; MyAvg += d; if (d < MyMin) MyMin = d; if (d > MyMax) MyMax = d; } A.Comm().MinAll(&MyMin, &GlobalMin, 1); A.Comm().MaxAll(&MyMax, &GlobalMax, 1); A.Comm().SumAll(&MyAvg, &GlobalAvg, 1); // does not really work fine if the number of global // elements is not a multiple of NumPDEEqns GlobalAvg /= (Diag.GlobalLength64() / NumPDEEqns); if (verbose) { char str[80]; sprintf(str, " A(k,k), eq %d", ie); print<double>(str, GlobalMin, GlobalAvg, GlobalMax); } } } // ======== // // row sums // // ======== // RowSum.MinValue(&GlobalMin); RowSum.MaxValue(&GlobalMax); RowSum.MeanValue(&GlobalAvg); if (verbose) { print(); print<double>(" sum_j A(k,j)", GlobalMin, GlobalAvg, GlobalMax); } // ===================================== // // cycle over all equations for row sums // // ===================================== // if (NumPDEEqns > 1 ) { if (verbose) print(); for (int ie = 0 ; ie < NumPDEEqns ; ie++) { MyMin = DBL_MAX; MyMax = -DBL_MAX; MyAvg = 0.0; for (int i = ie ; i < Diag.MyLength() ; i += NumPDEEqns) { double d = RowSum[i]; MyAvg += d; if (d < MyMin) MyMin = d; if (d > MyMax) MyMax = d; } A.Comm().MinAll(&MyMin, &GlobalMin, 1); A.Comm().MaxAll(&MyMax, &GlobalMax, 1); A.Comm().SumAll(&MyAvg, &GlobalAvg, 1); // does not really work fine if the number of global // elements is not a multiple of NumPDEEqns GlobalAvg /= (Diag.GlobalLength64() / NumPDEEqns); if (verbose) { char str[80]; sprintf(str, " sum_j A(k,j), eq %d", ie); print<double>(str, GlobalMin, GlobalAvg, GlobalMax); } } } if (verbose) Ifpack_PrintLine(); return(0); }
// ====================================================================== int Ifpack_PrintSparsity(const Epetra_RowMatrix& A, const char* InputFileName, const int NumPDEEqns) { int ltit; long long m,nc,nr,maxdim; double lrmrgn,botmrgn,xtit,ytit,ytitof,fnstit,siz = 0.0; double xl,xr, yb,yt, scfct,u2dot,frlw,delt,paperx; bool square = false; /*change square to .true. if you prefer a square frame around a rectangular matrix */ double conv = 2.54; char munt = 'E'; /* put 'E' for centimeters, 'U' for inches */ int ptitle = 0; /* position of the title, 0 under the drawing, else above */ FILE* fp = NULL; int NumMyRows; //int NumMyCols; long long NumGlobalRows; long long NumGlobalCols; int MyPID; int NumProc; char FileName[1024]; char title[1024]; const Epetra_Comm& Comm = A.Comm(); /* --------------------- execution begins ---------------------- */ if (strlen(A.Label()) != 0) strcpy(title, A.Label()); else sprintf(title, "%s", "matrix"); if (InputFileName == 0) sprintf(FileName, "%s.ps", title); else strcpy(FileName, InputFileName); MyPID = Comm.MyPID(); NumProc = Comm.NumProc(); NumMyRows = A.NumMyRows(); //NumMyCols = A.NumMyCols(); NumGlobalRows = A.NumGlobalRows64(); NumGlobalCols = A.NumGlobalCols64(); if (NumGlobalRows != NumGlobalCols) IFPACK_CHK_ERR(-1); // never tested /* to be changed for rect matrices */ maxdim = (NumGlobalRows>NumGlobalCols)?NumGlobalRows:NumGlobalCols; maxdim /= NumPDEEqns; m = 1 + maxdim; nr = NumGlobalRows / NumPDEEqns + 1; nc = NumGlobalCols / NumPDEEqns + 1; if (munt == 'E') { u2dot = 72.0/conv; paperx = 21.0; siz = 10.0; } else { u2dot = 72.0; paperx = 8.5*conv; siz = siz*conv; } /* left and right margins (drawing is centered) */ lrmrgn = (paperx-siz)/2.0; /* bottom margin : 2 cm */ botmrgn = 2.0; /* c scaling factor */ scfct = siz*u2dot/m; /* matrix frame line witdh */ frlw = 0.25; /* font size for title (cm) */ fnstit = 0.5; /* mfh 23 Jan 2013: title is always nonnull, since it's an array of fixed nonzero length. The 'if' test thus results in a compiler warning. */ /*if (title) ltit = strlen(title);*/ /*else ltit = 0;*/ ltit = strlen(title); /* position of title : centered horizontally */ /* at 1.0 cm vertically over the drawing */ ytitof = 1.0; xtit = paperx/2.0; ytit = botmrgn+siz*nr/m + ytitof; /* almost exact bounding box */ xl = lrmrgn*u2dot - scfct*frlw/2; xr = (lrmrgn+siz)*u2dot + scfct*frlw/2; yb = botmrgn*u2dot - scfct*frlw/2; yt = (botmrgn+siz*nr/m)*u2dot + scfct*frlw/2; if (ltit == 0) { yt = yt + (ytitof+fnstit*0.70)*u2dot; } /* add some room to bounding box */ delt = 10.0; xl = xl-delt; xr = xr+delt; yb = yb-delt; yt = yt+delt; /* correction for title under the drawing */ if ((ptitle == 0) && (ltit == 0)) { ytit = botmrgn + fnstit*0.3; botmrgn = botmrgn + ytitof + fnstit*0.7; } /* begin of output */ if (MyPID == 0) { fp = fopen(FileName,"w"); fprintf(fp,"%s","%%!PS-Adobe-2.0\n"); fprintf(fp,"%s","%%Creator: IFPACK\n"); fprintf(fp,"%%%%BoundingBox: %f %f %f %f\n", xl,yb,xr,yt); fprintf(fp,"%s","%%EndComments\n"); fprintf(fp,"%s","/cm {72 mul 2.54 div} def\n"); fprintf(fp,"%s","/mc {72 div 2.54 mul} def\n"); fprintf(fp,"%s","/pnum { 72 div 2.54 mul 20 string "); fprintf(fp,"%s","cvs print ( ) print} def\n"); fprintf(fp,"%s","/Cshow {dup stringwidth pop -2 div 0 rmoveto show} def\n"); /* we leave margins etc. in cm so it is easy to modify them if needed by editing the output file */ fprintf(fp,"%s","gsave\n"); if (ltit != 0) { fprintf(fp,"/Helvetica findfont %e cm scalefont setfont\n", fnstit); fprintf(fp,"%f cm %f cm moveto\n", xtit,ytit); fprintf(fp,"(%s) Cshow\n", title); fprintf(fp,"%f cm %f cm translate\n", lrmrgn,botmrgn); } fprintf(fp,"%f cm %d div dup scale \n", siz, (int) m); /* draw a frame around the matrix */ fprintf(fp,"%f setlinewidth\n", frlw); fprintf(fp,"%s","newpath\n"); fprintf(fp,"%s","0 0 moveto "); if (square) { printf("------------------- %d\n", (int) m); fprintf(fp,"%d %d lineto\n", (int) m, 0); fprintf(fp,"%d %d lineto\n", (int) m, (int) m); fprintf(fp,"%d %d lineto\n", 0, (int) m); } else { fprintf(fp,"%d %d lineto\n", (int) nc, 0); fprintf(fp,"%d %d lineto\n", (int) nc, (int) nr); fprintf(fp,"%d %d lineto\n", 0, (int) nr); } fprintf(fp,"%s","closepath stroke\n"); /* plotting loop */ fprintf(fp,"%s","1 1 translate\n"); fprintf(fp,"%s","0.8 setlinewidth\n"); fprintf(fp,"%s","/p {moveto 0 -.40 rmoveto \n"); fprintf(fp,"%s"," 0 .80 rlineto stroke} def\n"); fclose(fp); } int MaxEntries = A.MaxNumEntries(); std::vector<int> Indices(MaxEntries); std::vector<double> Values(MaxEntries); for (int pid = 0 ; pid < NumProc ; ++pid) { if (pid == MyPID) { fp = fopen(FileName,"a"); if( fp == NULL ) { fprintf(stderr,"%s","ERROR\n"); exit(EXIT_FAILURE); } for (int i = 0 ; i < NumMyRows ; ++i) { if (i % NumPDEEqns) continue; int Nnz; A.ExtractMyRowCopy(i,MaxEntries,Nnz,&Values[0],&Indices[0]); long long grow = A.RowMatrixRowMap().GID64(i); for (int j = 0 ; j < Nnz ; ++j) { int col = Indices[j]; if (col % NumPDEEqns == 0) { long long gcol = A.RowMatrixColMap().GID64(Indices[j]); grow /= NumPDEEqns; gcol /= NumPDEEqns; fprintf(fp,"%lld %lld p\n", gcol, NumGlobalRows - grow - 1); } } } fprintf(fp,"%s","%end of data for this process\n"); if( pid == NumProc - 1 ) fprintf(fp,"%s","showpage\n"); fclose(fp); } Comm.Barrier(); } return(0); }
int Amesos_Scalapack::Solve() { if( debug_ == 1 ) std::cout << "Entering `Solve()'" << std::endl; NumSolve_++; Epetra_MultiVector *vecX = Problem_->GetLHS() ; Epetra_MultiVector *vecB = Problem_->GetRHS() ; // // Compute the number of right hands sides // (and check that X and B have the same shape) // int nrhs; if ( vecX == 0 ) { nrhs = 0 ; EPETRA_CHK_ERR( vecB != 0 ) ; } else { nrhs = vecX->NumVectors() ; EPETRA_CHK_ERR( vecB->NumVectors() != nrhs ) ; } Epetra_MultiVector *ScalapackB =0; Epetra_MultiVector *ScalapackX =0; // // Extract Scalapack versions of X and B // double *ScalapackXvalues ; Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); Time_->ResetStartTime(); // track time to broadcast vectors // // Copy B to the scalapack version of B // const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap(); Epetra_MultiVector *ScalapackXextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ; Epetra_MultiVector *ScalapackBextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ; Epetra_Import ImportToScalapack( *VectorMap_, OriginalMap ); ScalapackBextract->Import( *vecB, ImportToScalapack, Insert ) ; ScalapackB = ScalapackBextract ; ScalapackX = ScalapackXextract ; VecTime_ += Time_->ElapsedTime(); // // Call SCALAPACKs PDGETRS to perform the solve // int DescX[10]; ScalapackX->Scale(1.0, *ScalapackB) ; int ScalapackXlda ; Time_->ResetStartTime(); // tract time to solve // // Setup DescX // if( nrhs > nb_ ) { EPETRA_CHK_ERR( -2 ); } int Ierr[1] ; Ierr[0] = 0 ; const int zero = 0 ; const int one = 1 ; if ( iam_ < nprow_ * npcol_ ) { assert( ScalapackX->ExtractView( &ScalapackXvalues, &ScalapackXlda ) == 0 ) ; if ( false ) std::cout << "Amesos_Scalapack.cpp: " << __LINE__ << " ScalapackXlda = " << ScalapackXlda << " lda_ = " << lda_ << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << " myprow_ = " << myprow_ << " mypcol_ = " << mypcol_ << " iam_ = " << iam_ << std::endl ; if ( TwoD_distribution_ ) assert( mypcol_ >0 || EPETRA_MAX(ScalapackXlda,1) == lda_ ) ; DESCINIT_F77(DescX, &NumGlobalElements_, &nrhs, &nb_, &nb_, &zero, &zero, &ictxt_, &lda_, Ierr ) ; assert( Ierr[0] == 0 ) ; // // For the 1D data distribution, we factor the transposed // matrix, hence we must invert the sense of the transposition // char trans = 'N'; if ( TwoD_distribution_ ) { if ( UseTranspose() ) trans = 'T' ; } else { if ( ! UseTranspose() ) trans = 'T' ; } if ( nprow_ * npcol_ == 1 ) { DGETRS_F77(&trans, &NumGlobalElements_, &nrhs, &DenseA_[0], &lda_, &Ipiv_[0], ScalapackXvalues, &lda_, Ierr ) ; } else { PDGETRS_F77(&trans, &NumGlobalElements_, &nrhs, &DenseA_[0], &one, &one, DescA_, &Ipiv_[0], ScalapackXvalues, &one, &one, DescX, Ierr ) ; } } SolTime_ += Time_->ElapsedTime(); Time_->ResetStartTime(); // track time to broadcast vectors // // Copy X back to the original vector // Epetra_Import ImportFromScalapack( OriginalMap, *VectorMap_ ); vecX->Import( *ScalapackX, ImportFromScalapack, Insert ) ; delete ScalapackBextract ; delete ScalapackXextract ; VecTime_ += Time_->ElapsedTime(); // All processes should return the same error code if ( nprow_ * npcol_ < Comm().NumProc() ) Comm().Broadcast( Ierr, 1, 0 ) ; // MS // compute vector norms if( ComputeVectorNorms_ == true || verbose_ == 2 ) { double NormLHS, NormRHS; for( int i=0 ; i<nrhs ; ++i ) { assert((*vecX)(i)->Norm2(&NormLHS)==0); assert((*vecB)(i)->Norm2(&NormRHS)==0); if( verbose_ && Comm().MyPID() == 0 ) { std::cout << "Amesos_Scalapack : vector " << i << ", ||x|| = " << NormLHS << ", ||b|| = " << NormRHS << std::endl; } } } // MS // compute true residual if( ComputeTrueResidual_ == true || verbose_ == 2 ) { double Norm; Epetra_MultiVector Ax(vecB->Map(),nrhs); for( int i=0 ; i<nrhs ; ++i ) { (Problem_->GetMatrix()->Multiply(UseTranspose(), *((*vecX)(i)), Ax)); (Ax.Update(1.0, *((*vecB)(i)), -1.0)); (Ax.Norm2(&Norm)); if( verbose_ && Comm().MyPID() == 0 ) { std::cout << "Amesos_Scalapack : vector " << i << ", ||Ax - b|| = " << Norm << std::endl; } } } return Ierr[0]; }
int DoCopyRowMatrix(mxArray* matlabA, int& valueCount, const Epetra_RowMatrix& A) { //cout << "doing DoCopyRowMatrix\n"; int ierr = 0; int numRows = A.NumGlobalRows(); //cout << "numRows: " << numRows << "\n"; Epetra_Map rowMap = A.RowMatrixRowMap(); Epetra_Map colMap = A.RowMatrixColMap(); int minAllGID = rowMap.MinAllGID(); const Epetra_Comm & comm = rowMap.Comm(); //cout << "did global setup\n"; if (comm.MyPID()!=0) { if (A.NumMyRows()!=0) ierr = -1; if (A.NumMyCols()!=0) ierr = -1; } else { // declare and get initial values of all matlabA pointers double* matlabAvaluesPtr = mxGetPr(matlabA); int* matlabAcolumnIndicesPtr = mxGetJc(matlabA); int* matlabArowIndicesPtr = mxGetIr(matlabA); // set all matlabA pointers to the proper offset matlabAvaluesPtr += valueCount; matlabArowIndicesPtr += valueCount; if (numRows!=A.NumMyRows()) ierr = -1; Epetra_SerialDenseVector values(A.MaxNumEntries()); Epetra_IntSerialDenseVector indices(A.MaxNumEntries()); //cout << "did proc0 setup\n"; for (int i=0; i<numRows; i++) { //cout << "extracting a row\n"; int I = rowMap.GID(i); int numEntries = 0; if (A.ExtractMyRowCopy(i, values.Length(), numEntries, values.Values(), indices.Values())) return(-1); matlabAcolumnIndicesPtr[I - minAllGID] = valueCount; // set the starting index of column I double* serialValuesPtr = values.Values(); for (int j=0; j<numEntries; j++) { int J = colMap.GID(indices[j]); *matlabAvaluesPtr = *serialValuesPtr++; *matlabArowIndicesPtr = J; // increment matlabA pointers matlabAvaluesPtr++; matlabArowIndicesPtr++; valueCount++; } } //cout << "proc0 row extraction for this chunck is done\n"; } /* if (comm.MyPID() == 0) { cout << "printing matlabA pointers\n"; double* matlabAvaluesPtr = mxGetPr(matlabA); int* matlabAcolumnIndicesPtr = mxGetJc(matlabA); int* matlabArowIndicesPtr = mxGetIr(matlabA); for(int i=0; i < numRows; i++) { for(int j=0; j < A.MaxNumEntries(); j++) { cout << "*matlabAvaluesPtr: " << *matlabAvaluesPtr++ << " *matlabAcolumnIndicesPtr: " << *matlabAcolumnIndicesPtr++ << " *matlabArowIndicesPtr" << *matlabArowIndicesPtr++ << "\n"; } } cout << "done printing matlabA pointers\n"; } */ int ierrGlobal; comm.MinAll(&ierr, &ierrGlobal, 1); // If any processor has -1, all return -1 return(ierrGlobal); }
//============================================================================= int Amesos_Mumps::ConvertToTriplet(const bool OnlyValues) { Epetra_RowMatrix* ptr; if (Comm().NumProc() == 1) ptr = &Matrix(); else { ptr = &RedistrMatrix(true); } ResetTimer(); #ifdef EXTRA_DEBUG_INFO Epetra_CrsMatrix* Eptr = dynamic_cast<Epetra_CrsMatrix*>( ptr ); if ( ptr->NumGlobalNonzeros() < 300 ) SetICNTL(4,3 ); // Enable more debug info for small matrices if ( ptr->NumGlobalNonzeros() < 42 && Eptr ) { std::cout << " Matrix = " << std::endl ; Eptr->Print( std::cout ) ; } else { assert( Eptr ); } #endif Row.resize(ptr->NumMyNonzeros()); Col.resize(ptr->NumMyNonzeros()); Val.resize(ptr->NumMyNonzeros()); int MaxNumEntries = ptr->MaxNumEntries(); std::vector<int> Indices; std::vector<double> Values; Indices.resize(MaxNumEntries); Values.resize(MaxNumEntries); int count = 0; for (int i = 0; i < ptr->NumMyRows() ; ++i) { int GlobalRow = ptr->RowMatrixRowMap().GID(i); int NumEntries = 0; int ierr; ierr = ptr->ExtractMyRowCopy(i, MaxNumEntries, NumEntries, &Values[0], &Indices[0]); AMESOS_CHK_ERR(ierr); for (int j = 0 ; j < NumEntries ; ++j) { if (OnlyValues == false) { Row[count] = GlobalRow + 1; Col[count] = ptr->RowMatrixColMap().GID(Indices[j]) + 1; } // MS // Added on 15-Mar-05. if (AddToDiag_ && Indices[j] == i) Values[j] += AddToDiag_; Val[count] = Values[j]; count++; } } MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_); assert (count <= ptr->NumMyNonzeros()); return(0); }
int check(Epetra_RowMatrix& A, Epetra_RowMatrix & B, bool verbose) { int ierr = 0; EPETRA_TEST_ERR(!A.Comm().NumProc()==B.Comm().NumProc(),ierr); EPETRA_TEST_ERR(!A.Comm().MyPID()==B.Comm().MyPID(),ierr); EPETRA_TEST_ERR(!A.Filled()==B.Filled(),ierr); EPETRA_TEST_ERR(!A.HasNormInf()==B.HasNormInf(),ierr); EPETRA_TEST_ERR(!A.LowerTriangular()==B.LowerTriangular(),ierr); EPETRA_TEST_ERR(!A.Map().SameAs(B.Map()),ierr); EPETRA_TEST_ERR(!A.MaxNumEntries()==B.MaxNumEntries(),ierr); EPETRA_TEST_ERR(!A.NumGlobalCols64()==B.NumGlobalCols64(),ierr); EPETRA_TEST_ERR(!A.NumGlobalDiagonals64()==B.NumGlobalDiagonals64(),ierr); EPETRA_TEST_ERR(!A.NumGlobalNonzeros64()==B.NumGlobalNonzeros64(),ierr); EPETRA_TEST_ERR(!A.NumGlobalRows64()==B.NumGlobalRows64(),ierr); EPETRA_TEST_ERR(!A.NumMyCols()==B.NumMyCols(),ierr); EPETRA_TEST_ERR(!A.NumMyDiagonals()==B.NumMyDiagonals(),ierr); EPETRA_TEST_ERR(!A.NumMyNonzeros()==B.NumMyNonzeros(),ierr); for (int i=0; i<A.NumMyRows(); i++) { int nA, nB; A.NumMyRowEntries(i,nA); B.NumMyRowEntries(i,nB); EPETRA_TEST_ERR(!nA==nB,ierr); } EPETRA_TEST_ERR(!A.NumMyRows()==B.NumMyRows(),ierr); EPETRA_TEST_ERR(!A.OperatorDomainMap().SameAs(B.OperatorDomainMap()),ierr); EPETRA_TEST_ERR(!A.OperatorRangeMap().SameAs(B.OperatorRangeMap()),ierr); EPETRA_TEST_ERR(!A.RowMatrixColMap().SameAs(B.RowMatrixColMap()),ierr); EPETRA_TEST_ERR(!A.RowMatrixRowMap().SameAs(B.RowMatrixRowMap()),ierr); EPETRA_TEST_ERR(!A.UpperTriangular()==B.UpperTriangular(),ierr); EPETRA_TEST_ERR(!A.UseTranspose()==B.UseTranspose(),ierr); int NumVectors = 5; { // No transpose case Epetra_MultiVector X(A.OperatorDomainMap(), NumVectors); Epetra_MultiVector YA1(A.OperatorRangeMap(), NumVectors); Epetra_MultiVector YA2(YA1); Epetra_MultiVector YB1(YA1); Epetra_MultiVector YB2(YA1); X.Random(); bool transA = false; A.SetUseTranspose(transA); B.SetUseTranspose(transA); A.Apply(X,YA1); A.Multiply(transA, X, YA2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2,"A Multiply and A Apply", verbose),ierr); B.Apply(X,YB1); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1,"A Multiply and B Multiply", verbose),ierr); B.Multiply(transA, X, YB2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2,"A Multiply and B Apply", verbose), ierr); } {// transpose case Epetra_MultiVector X(A.OperatorRangeMap(), NumVectors); Epetra_MultiVector YA1(A.OperatorDomainMap(), NumVectors); Epetra_MultiVector YA2(YA1); Epetra_MultiVector YB1(YA1); Epetra_MultiVector YB2(YA1); X.Random(); bool transA = true; A.SetUseTranspose(transA); B.SetUseTranspose(transA); A.Apply(X,YA1); A.Multiply(transA, X, YA2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2, "A Multiply and A Apply (transpose)", verbose),ierr); B.Apply(X,YB1); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1, "A Multiply and B Multiply (transpose)", verbose),ierr); B.Multiply(transA, X,YB2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2, "A Multiply and B Apply (transpose)", verbose),ierr); } Epetra_Vector diagA(A.RowMatrixRowMap()); EPETRA_TEST_ERR(A.ExtractDiagonalCopy(diagA),ierr); Epetra_Vector diagB(B.RowMatrixRowMap()); EPETRA_TEST_ERR(B.ExtractDiagonalCopy(diagB),ierr); EPETRA_TEST_ERR(checkMultiVectors(diagA,diagB, "ExtractDiagonalCopy", verbose),ierr); Epetra_Vector rowA(A.RowMatrixRowMap()); EPETRA_TEST_ERR(A.InvRowSums(rowA),ierr); Epetra_Vector rowB(B.RowMatrixRowMap()); EPETRA_TEST_ERR(B.InvRowSums(rowB),ierr) EPETRA_TEST_ERR(checkMultiVectors(rowA,rowB, "InvRowSums", verbose),ierr); Epetra_Vector colA(A.RowMatrixColMap()); EPETRA_TEST_ERR(A.InvColSums(colA),ierr); Epetra_Vector colB(B.RowMatrixColMap()); EPETRA_TEST_ERR(B.InvColSums(colB),ierr); EPETRA_TEST_ERR(checkMultiVectors(colA,colB, "InvColSums", verbose),ierr); EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf before scaling", verbose), ierr); EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne before scaling", verbose),ierr); EPETRA_TEST_ERR(A.RightScale(colA),ierr); EPETRA_TEST_ERR(B.RightScale(colB),ierr); EPETRA_TEST_ERR(A.LeftScale(rowA),ierr); EPETRA_TEST_ERR(B.LeftScale(rowB),ierr); EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf after scaling", verbose), ierr); EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne after scaling", verbose),ierr); vector<double> valuesA(A.MaxNumEntries()); vector<int> indicesA(A.MaxNumEntries()); vector<double> valuesB(B.MaxNumEntries()); vector<int> indicesB(B.MaxNumEntries()); return(0); for (int i=0; i<A.NumMyRows(); i++) { int nA, nB; EPETRA_TEST_ERR(A.ExtractMyRowCopy(i, A.MaxNumEntries(), nA, &valuesA[0], &indicesA[0]),ierr); EPETRA_TEST_ERR(B.ExtractMyRowCopy(i, B.MaxNumEntries(), nB, &valuesB[0], &indicesB[0]),ierr); EPETRA_TEST_ERR(!nA==nB,ierr); for (int j=0; j<nA; j++) { double curVal = valuesA[j]; int curIndex = indicesA[j]; bool notfound = true; int jj = 0; while (notfound && jj< nB) { if (!checkValues(curVal, valuesB[jj])) notfound = false; jj++; } EPETRA_TEST_ERR(notfound, ierr); vector<int>::iterator p = find(indicesB.begin(),indicesB.end(),curIndex); // find curIndex in indicesB EPETRA_TEST_ERR(p==indicesB.end(), ierr); } } if (verbose) cout << "RowMatrix Methods check OK" << endl; return (ierr); }
// ====================================================================== int GetAggregates(Epetra_RowMatrix& A, Teuchos::ParameterList& List, double* thisns, Epetra_IntVector& aggrinfo) { if (!A.RowMatrixRowMap().SameAs(aggrinfo.Map())) ML_THROW("map of aggrinfo must match row map of operator", -1); std::string CoarsenType = List.get("aggregation: type", "Uncoupled"); double Threshold = List.get("aggregation: threshold", 0.0); int NumPDEEquations = List.get("PDE equations", 1); int nsdim = List.get("null space: dimension",-1); if (nsdim==-1) ML_THROW("dimension of nullspace not set", -1); int size = A.RowMatrixRowMap().NumMyElements(); ML_Aggregate* agg_object; ML_Aggregate_Create(&agg_object); ML_Aggregate_KeepInfo(agg_object,1); ML_Aggregate_Set_MaxLevels(agg_object,2); ML_Aggregate_Set_StartLevel(agg_object,0); ML_Aggregate_Set_Threshold(agg_object,Threshold); //agg_object->curr_threshold = 0.0; ML_Operator* ML_Ptent = 0; ML_Ptent = ML_Operator_Create(GetML_Comm()); if (!thisns) ML_THROW("nullspace is NULL", -1); ML_Aggregate_Set_NullSpace(agg_object, NumPDEEquations, nsdim, thisns,size); if (CoarsenType == "Uncoupled") agg_object->coarsen_scheme = ML_AGGR_UNCOUPLED; else if (CoarsenType == "Uncoupled-MIS") agg_object->coarsen_scheme = ML_AGGR_HYBRIDUM; else if (CoarsenType == "MIS") { /* needed for MIS, otherwise it sets the number of equations to * the null space dimension */ agg_object->max_levels = -7; agg_object->coarsen_scheme = ML_AGGR_MIS; } else if (CoarsenType == "METIS") agg_object->coarsen_scheme = ML_AGGR_METIS; else { ML_THROW("Requested aggregation scheme (" + CoarsenType + ") not recognized", -1); } ML_Operator* ML_A = ML_Operator_Create(GetML_Comm()); ML_Operator_WrapEpetraMatrix(&A,ML_A); int NextSize = ML_Aggregate_Coarsen(agg_object, ML_A, &ML_Ptent, GetML_Comm()); int* aggrmap = NULL; ML_Aggregate_Get_AggrMap(agg_object,0,&aggrmap); if (!aggrmap) ML_THROW("aggr_info not available", -1); #if 0 // debugging fflush(stdout); for (int proc=0; proc<A.GetRowMatrix()->Comm().NumProc(); ++proc) { if (A.GetRowMatrix()->Comm().MyPID()==proc) { std::cout << "Proc " << proc << ":" << std::endl; std::cout << "aggrcount " << aggrcount << std::endl; std::cout << "NextSize " << NextSize << std::endl; for (int i=0; i<size; ++i) std::cout << "aggrmap[" << i << "] = " << aggrmap[i] << std::endl; fflush(stdout); } A.GetRowMatrix()->Comm().Barrier(); } #endif assert (NextSize * nsdim != 0); for (int i=0; i<size; ++i) aggrinfo[i] = aggrmap[i]; ML_Aggregate_Destroy(&agg_object); return (NextSize/nsdim); }
//============================================================================= int Amesos_Dscpack::PerformNumericFactorization() { ResetTimer(0); ResetTimer(1); Epetra_RowMatrix* RowMatrixA = Problem_->GetMatrix(); if (RowMatrixA == 0) AMESOS_CHK_ERR(-1); const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ; int numrows = RowMatrixA->NumGlobalRows(); assert( numrows == RowMatrixA->NumGlobalCols() ); // // Call Dscpack to perform Numeric Factorization // std::vector<double> MyANonZ; #if 0 if ( IsNumericFactorizationOK_ ) { DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject); } #endif DscRowMap_ = Teuchos::rcp(new Epetra_Map(numrows, NumLocalCols, LocalStructOldNum, 0, Comm())); if (DscRowMap_.get() == 0) AMESOS_CHK_ERR(-1); Importer_ = rcp(new Epetra_Import(DscRowMap(), OriginalMap)); // // Import from the CrsMatrix // Epetra_CrsMatrix DscMat(Copy, DscRowMap(), 0); AMESOS_CHK_ERR(DscMat.Import(*RowMatrixA, Importer(), Insert)); AMESOS_CHK_ERR(DscMat.FillComplete()); DscColMap_ = Teuchos::rcp(new Epetra_Map(DscMat.RowMatrixColMap())); assert( MyDscRank >= 0 || NumLocalNonz == 0 ) ; assert( MyDscRank >= 0 || NumLocalCols == 0 ) ; assert( MyDscRank >= 0 || NumGlobalCols == 0 ) ; MyANonZ.resize( NumLocalNonz ) ; int NonZIndex = 0 ; int max_num_entries = DscMat.MaxNumEntries() ; std::vector<int> col_indices( max_num_entries ) ; std::vector<double> mat_values( max_num_entries ) ; assert( NumLocalCols == DscRowMap().NumMyElements() ) ; std::vector<int> my_global_elements( NumLocalCols ) ; AMESOS_CHK_ERR(DscRowMap().MyGlobalElements( &my_global_elements[0] ) ) ; std::vector<int> GlobalStructOldColNum( NumGlobalCols ) ; typedef std::pair<int, double> Data; std::vector<Data> sort_array(max_num_entries); std::vector<int> sort_indices(max_num_entries); for ( int i = 0; i < NumLocalCols ; i++ ) { assert( my_global_elements[i] == LocalStructOldNum[i] ) ; int num_entries_this_row; #ifdef USE_LOCAL AMESOS_CHK_ERR( DscMat.ExtractMyRowCopy( i, max_num_entries, num_entries_this_row, &mat_values[0], &col_indices[0] ) ) ; #else AMESOS_CHK_ERR( DscMat.ExtractGlobalRowCopy( DscMat.GRID(i), max_num_entries, num_entries_this_row, &mat_values[0], &col_indices[0] ) ) ; #endif int OldRowNumber = LocalStructOldNum[i] ; if (GlobalStructOwner[ OldRowNumber ] == -1) AMESOS_CHK_ERR(-1); int NewRowNumber = GlobalStructNewColNum[ my_global_elements[ i ] ] ; // // Sort the column elements // for ( int j = 0; j < num_entries_this_row; j++ ) { #ifdef USE_LOCAL sort_array[j].first = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ; sort_indices[j] = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ; #else sort_array[j].first = GlobalStructNewColNum[ col_indices[j] ]; #endif sort_array[j].second = mat_values[j] ; } sort(&sort_array[0], &sort_array[num_entries_this_row]); for ( int j = 0; j < num_entries_this_row; j++ ) { int NewColNumber = sort_array[j].first ; if ( NewRowNumber <= NewColNumber ) MyANonZ[ NonZIndex++ ] = sort_array[j].second ; #ifndef USE_LOCAL assert( NonZIndex <= NumLocalNonz ); // This assert can fail on non-symmetric matrices #endif } } OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); if ( MyDscRank >= 0 ) { const int SchemeCode = 1; #ifndef USE_LOCAL assert( NonZIndex == NumLocalNonz ); #endif AMESOS_CHK_ERR( DSC_NFactor ( PrivateDscpackData_->MyDSCObject_, SchemeCode, &MyANonZ[0], DSC_LLT, DSC_LBLAS3, DSC_DBLAS2 ) ) ; } // if ( MyDscRank >= 0 ) IsNumericFactorizationOK_ = true ; NumFactTime_ = AddTime("Total numeric factorization time", NumFactTime_, 0); return(0); }
// ====================================================================== void GetPtent(const Epetra_RowMatrix& A, Teuchos::ParameterList& List, double* thisns, Teuchos::RCP<Epetra_CrsMatrix>& Ptent, Teuchos::RCP<Epetra_MultiVector>& NextNS, const int domainoffset) { const int nsdim = List.get<int>("null space: dimension",-1); if (nsdim<=0) ML_THROW("null space dimension not given",-1); const Epetra_Map& rowmap = A.RowMatrixRowMap(); const int mylength = rowmap.NumMyElements(); // wrap nullspace into something more handy Epetra_MultiVector ns(View,rowmap,thisns,mylength,nsdim); // vector to hold aggregation information Epetra_IntVector aggs(rowmap,false); // aggregation with global aggregate numbering int naggregates = GetGlobalAggregates(const_cast<Epetra_RowMatrix&>(A),List,thisns,aggs); // build a domain map for Ptent // find first aggregate on proc int firstagg = -1; int offset = -1; for (int i=0; i<mylength; ++i) if (aggs[i]>=0) { offset = firstagg = aggs[i]; break; } offset *= nsdim; if (offset<0) ML_THROW("could not find any aggregate on proc",-2); std::vector<int> coarsegids(naggregates*nsdim); for (int i=0; i<naggregates; ++i) for (int j=0; j<nsdim; ++j) { coarsegids[i*nsdim+j] = offset + domainoffset; ++offset; } Epetra_Map pdomainmap(-1,naggregates*nsdim,&coarsegids[0],0,A.Comm()); // loop aggregates and build ids for dofs std::map<int,std::vector<int> > aggdofs; std::map<int,std::vector<int> >::iterator fool; for (int i=0; i<naggregates; ++i) { std::vector<int> gids(0); aggdofs.insert(std::pair<int,std::vector<int> >(firstagg+i,gids)); } for (int i=0; i<mylength; ++i) { if (aggs[i]<0) continue; std::vector<int>& gids = aggdofs[aggs[i]]; gids.push_back(aggs.Map().GID(i)); } #if 0 // debugging output for (int proc=0; proc<A.Comm().NumProc(); ++proc) { if (proc==A.Comm().MyPID()) { for (fool=aggdofs.begin(); fool!=aggdofs.end(); ++fool) { std::cout << "Proc " << proc << " Aggregate " << fool->first << " Dofs "; std::vector<int>& gids = fool->second; for (int i=0; i<(int)gids.size(); ++i) std::cout << gids[i] << " "; std::cout << std::endl; } } fflush(stdout); A.Comm().Barrier(); } #endif // coarse level nullspace to be filled NextNS = Teuchos::rcp(new Epetra_MultiVector(pdomainmap,nsdim,true)); Epetra_MultiVector& nextns = *NextNS; // matrix Ptent Ptent = Teuchos::rcp(new Epetra_CrsMatrix(Copy,rowmap,nsdim)); // loop aggregates and extract the appropriate slices of the nullspace. // do QR and assemble Q and R to Ptent and NextNS for (fool=aggdofs.begin(); fool!=aggdofs.end(); ++fool) { // extract aggregate-local junk of nullspace const int aggsize = (int)fool->second.size(); Epetra_SerialDenseMatrix Bagg(aggsize,nsdim); for (int i=0; i<aggsize; ++i) for (int j=0; j<nsdim; ++j) Bagg(i,j) = (*ns(j))[ns.Map().LID(fool->second[i])]; // Bagg = Q*R int m = Bagg.M(); int n = Bagg.N(); int lwork = n*10; int info = 0; int k = std::min(m,n); if (k!=n) ML_THROW("Aggregate too small, fatal!",-1); std::vector<double> work(lwork); std::vector<double> tau(k); Epetra_LAPACK lapack; lapack.GEQRF(m,n,Bagg.A(),m,&tau[0],&work[0],lwork,&info); if (info) ML_THROW("Lapack dgeqrf returned nonzero",info); if (work[0]>lwork) { lwork = (int)work[0]; work.resize(lwork); } // get R (stored on Bagg) and assemble it into nextns int agg_cgid = fool->first*nsdim; if (!nextns.Map().MyGID(agg_cgid+domainoffset)) ML_THROW("Missing coarse column id on this proc",-1); for (int i=0; i<n; ++i) for (int j=i; j<n; ++j) (*nextns(j))[nextns.Map().LID(domainoffset+agg_cgid+i)] = Bagg(i,j); // get Q and assemble it into Ptent lapack.ORGQR(m,n,k,Bagg.A(),m,&tau[0],&work[0],lwork,&info); if (info) ML_THROW("Lapack dorgqr returned nonzero",info); for (int i=0; i<aggsize; ++i) { const int actgrow = fool->second[i]; for (int j=0; j<nsdim; ++j) { int actgcol = fool->first*nsdim+j+domainoffset; int errone = Ptent->SumIntoGlobalValues(actgrow,1,&Bagg(i,j),&actgcol); if (errone>0) { int errtwo = Ptent->InsertGlobalValues(actgrow,1,&Bagg(i,j),&actgcol); if (errtwo<0) ML_THROW("Epetra_CrsMatrix::InsertGlobalValues returned negative nonzero",errtwo); } else if (errone) ML_THROW("Epetra_CrsMatrix::SumIntoGlobalValues returned negative nonzero",errone); } } // for (int i=0; i<aggsize; ++i) } // for (fool=aggdofs.begin(); fool!=aggdofs.end(); ++fool) int err = Ptent->FillComplete(pdomainmap,rowmap); if (err) ML_THROW("Epetra_CrsMatrix::FillComplete returned nonzero",err); err = Ptent->OptimizeStorage(); if (err) ML_THROW("Epetra_CrsMatrix::OptimizeStorage returned nonzero",err); return; }