void ModeLaplace1DQ1::memoryInfo() const { int myPid = MyComm.MyPID(); Epetra_RowMatrix *Mat = dynamic_cast<Epetra_RowMatrix *>(M); if ((myPid == 0) && (Mat)) { cout << " Total number of nonzero entries in mass matrix = "; cout.width(15); cout << Mat->NumGlobalNonzeros() << endl; double memSize = Mat->NumGlobalNonzeros()*(sizeof(double) + sizeof(int)); memSize += 2*Mat->NumGlobalRows()*sizeof(int); cout << " Memory requested for mass matrix per processor = (EST) "; cout.precision(2); cout.width(6); cout.setf(ios::fixed, ios::floatfield); cout << memSize/1024.0/1024.0/MyComm.NumProc() << " MB " << endl; cout << endl; } Mat = dynamic_cast<Epetra_RowMatrix *>(K); if ((myPid == 0) && (Mat)) { cout << " Total number of nonzero entries in stiffness matrix = "; cout.width(15); cout << Mat->NumGlobalNonzeros() << endl; double memSize = Mat->NumGlobalNonzeros()*(sizeof(double) + sizeof(int)); memSize += 2*Mat->NumGlobalRows()*sizeof(int); cout << " Memory requested for stiffness matrix per processor = (EST) "; cout.precision(2); cout.width(6); cout.setf(ios::fixed, ios::floatfield); cout << memSize/1024.0/1024.0/MyComm.NumProc() << " MB " << endl; cout << endl; } }
// ============================================================================ void EpetraExt::XMLWriter:: Write(const std::string& Label, const Epetra_RowMatrix& Matrix) { TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error, "No file has been opened"); int Rows = Matrix.NumGlobalRows(); int Cols = Matrix.NumGlobalRows(); int Nonzeros = Matrix.NumGlobalNonzeros(); if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "<PointMatrix Label=\"" << Label << '"' << " Rows=\"" << Rows << '"' << " Columns=\"" << Cols<< '"' << " Nonzeros=\"" << Nonzeros << '"' << " Type=\"double\" StartingIndex=\"0\">" << std::endl; } int Length = Matrix.MaxNumEntries(); std::vector<int> Indices(Length); std::vector<double> Values(Length); for (int iproc = 0; iproc < Comm_.NumProc(); iproc++) { if (iproc == Comm_.MyPID()) { std::ofstream of(FileName_.c_str(), std::ios::app); of.precision(15); for (int i = 0; i < Matrix.NumMyRows(); ++i) { int NumMyEntries; Matrix.ExtractMyRowCopy(i, Length, NumMyEntries, &Values[0], &Indices[0]); int GRID = Matrix.RowMatrixRowMap().GID(i); for (int j = 0; j < NumMyEntries; ++j) of << GRID << " " << Matrix.RowMatrixColMap().GID(Indices[j]) << " " << std::setiosflags(std::ios::scientific) << Values[j] << std::endl; } of.close(); } Comm_.Barrier(); } if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "</PointMatrix>" << std::endl; of.close(); } }
int Amesos_Scalapack::NumericFactorization() { if( debug_ == 1 ) std::cout << "Entering `NumericFactorization()'" << std::endl; NumNumericFact_++; iam_ = Comm().MyPID(); Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; NumGlobalElements_ = OriginalMap.NumGlobalElements(); NumGlobalNonzeros_ = RowMatrixA->NumGlobalNonzeros(); RedistributeA(); ConvertToScalapack(); return PerformNumericFactorization( ); }
//============================================================================= int Amesos_Dscpack::PerformSymbolicFactorization() { ResetTimer(0); ResetTimer(1); MyPID_ = Comm().MyPID(); NumProcs_ = Comm().NumProc(); Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix(); if (RowMatrixA == 0) AMESOS_CHK_ERR(-1); const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ; const Epetra_MpiComm& comm1 = dynamic_cast<const Epetra_MpiComm &> (Comm()); int numrows = RowMatrixA->NumGlobalRows(); int numentries = RowMatrixA->NumGlobalNonzeros(); Teuchos::RCP<Epetra_CrsGraph> Graph; Epetra_CrsMatrix* CastCrsMatrixA = dynamic_cast<Epetra_CrsMatrix*>(RowMatrixA); if (CastCrsMatrixA) { Graph = Teuchos::rcp(const_cast<Epetra_CrsGraph*>(&(CastCrsMatrixA->Graph())), false); } else { int MaxNumEntries = RowMatrixA->MaxNumEntries(); Graph = Teuchos::rcp(new Epetra_CrsGraph(Copy, OriginalMap, MaxNumEntries)); std::vector<int> Indices(MaxNumEntries); std::vector<double> Values(MaxNumEntries); for (int i = 0 ; i < RowMatrixA->NumMyRows() ; ++i) { int NumEntries; RowMatrixA->ExtractMyRowCopy(i, MaxNumEntries, NumEntries, &Values[0], &Indices[0]); for (int j = 0 ; j < NumEntries ; ++j) Indices[j] = RowMatrixA->RowMatrixColMap().GID(Indices[j]); int GlobalRow = RowMatrixA->RowMatrixRowMap().GID(i); Graph->InsertGlobalIndices(GlobalRow, NumEntries, &Indices[0]); } Graph->FillComplete(); } // // Create a replicated map and graph // std::vector<int> AllIDs( numrows ) ; for ( int i = 0; i < numrows ; i++ ) AllIDs[i] = i ; Epetra_Map ReplicatedMap( -1, numrows, &AllIDs[0], 0, Comm()); Epetra_Import ReplicatedImporter(ReplicatedMap, OriginalMap); Epetra_CrsGraph ReplicatedGraph( Copy, ReplicatedMap, 0 ); AMESOS_CHK_ERR(ReplicatedGraph.Import(*Graph, ReplicatedImporter, Insert)); AMESOS_CHK_ERR(ReplicatedGraph.FillComplete()); // // Convert the matrix to Ap, Ai // std::vector <int> Replicates(numrows); std::vector <int> Ap(numrows + 1); std::vector <int> Ai(EPETRA_MAX(numrows, numentries)); for( int i = 0 ; i < numrows; i++ ) Replicates[i] = 1; int NumEntriesPerRow ; int *ColIndices = 0 ; int Ai_index = 0 ; for ( int MyRow = 0; MyRow <numrows; MyRow++ ) { AMESOS_CHK_ERR( ReplicatedGraph.ExtractMyRowView( MyRow, NumEntriesPerRow, ColIndices ) ); Ap[MyRow] = Ai_index ; for ( int j = 0; j < NumEntriesPerRow; j++ ) { Ai[Ai_index] = ColIndices[j] ; Ai_index++; } } assert( Ai_index == numentries ) ; Ap[ numrows ] = Ai_index ; MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0); ResetTimer(0); // // Call Dscpack Symbolic Factorization // int OrderCode = 2; std::vector<double> MyANonZ; NumLocalNonz = 0 ; GlobalStructNewColNum = 0 ; GlobalStructNewNum = 0 ; GlobalStructOwner = 0 ; LocalStructOldNum = 0 ; NumGlobalCols = 0 ; // MS // Have to define the maximum number of processes to be used // MS // This is only a suggestion as Dscpack uses a number of processes that is a power of 2 int NumGlobalNonzeros = GetProblem()->GetMatrix()->NumGlobalNonzeros(); int NumRows = GetProblem()->GetMatrix()->NumGlobalRows(); // optimal value for MaxProcs == -1 int OptNumProcs1 = 1+EPETRA_MAX( NumRows/10000, NumGlobalNonzeros/1000000 ); OptNumProcs1 = EPETRA_MIN(NumProcs_,OptNumProcs1 ); // optimal value for MaxProcs == -2 int OptNumProcs2 = (int)sqrt(1.0 * NumProcs_); if( OptNumProcs2 < 1 ) OptNumProcs2 = 1; // fix the value of MaxProcs switch (MaxProcs_) { case -1: MaxProcs_ = EPETRA_MIN(OptNumProcs1, NumProcs_); break; case -2: MaxProcs_ = EPETRA_MIN(OptNumProcs2, NumProcs_); break; case -3: MaxProcs_ = NumProcs_; break; } #if 0 if (MyDscRank>=0 && A_and_LU_built) { DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject); } #endif // if ( ! A_and_LU_built ) { // DSC_End( PrivateDscpackData_->MyDSCObject ) ; // PrivateDscpackData_->MyDSCObject = DSC_Begin() ; // } // MS // here I continue with the old code... OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); DscNumProcs = 1 ; int DscMax = DSC_Analyze( numrows, &Ap[0], &Ai[0], &Replicates[0] ); while ( DscNumProcs * 2 <=EPETRA_MIN( MaxProcs_, DscMax ) ) DscNumProcs *= 2 ; MyDscRank = -1; DSC_Open0( PrivateDscpackData_->MyDSCObject_, DscNumProcs, &MyDscRank, comm1.Comm()) ; NumLocalCols = 0 ; // This is for those processes not in the Dsc grid if ( MyDscRank >= 0 ) { assert( MyPID_ == MyDscRank ) ; AMESOS_CHK_ERR( DSC_Order ( PrivateDscpackData_->MyDSCObject_, OrderCode, numrows, &Ap[0], &Ai[0], &Replicates[0], &NumGlobalCols, &NumLocalStructs, &NumLocalCols, &NumLocalNonz, &GlobalStructNewColNum, &GlobalStructNewNum, &GlobalStructOwner, &LocalStructOldNum ) ) ; assert( NumGlobalCols == numrows ) ; assert( NumLocalCols == NumLocalStructs ) ; } if ( MyDscRank >= 0 ) { int MaxSingleBlock; const int Limit = 5000000 ; // Memory Limit set to 5 Terabytes AMESOS_CHK_ERR( DSC_SFactor ( PrivateDscpackData_->MyDSCObject_, &TotalMemory_, &MaxSingleBlock, Limit, DSC_LBLAS3, DSC_DBLAS2 ) ) ; } // A_and_LU_built = true; // If you uncomment this, TestOptions fails SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_, 0); return(0); }
int check(Epetra_RowMatrix& A, Epetra_RowMatrix & B, bool verbose) { int ierr = 0; EPETRA_TEST_ERR(!A.Comm().NumProc()==B.Comm().NumProc(),ierr); EPETRA_TEST_ERR(!A.Comm().MyPID()==B.Comm().MyPID(),ierr); EPETRA_TEST_ERR(!A.Filled()==B.Filled(),ierr); EPETRA_TEST_ERR(!A.HasNormInf()==B.HasNormInf(),ierr); EPETRA_TEST_ERR(!A.LowerTriangular()==B.LowerTriangular(),ierr); EPETRA_TEST_ERR(!A.Map().SameAs(B.Map()),ierr); EPETRA_TEST_ERR(!A.MaxNumEntries()==B.MaxNumEntries(),ierr); EPETRA_TEST_ERR(!A.NumGlobalCols()==B.NumGlobalCols(),ierr); EPETRA_TEST_ERR(!A.NumGlobalDiagonals()==B.NumGlobalDiagonals(),ierr); EPETRA_TEST_ERR(!A.NumGlobalNonzeros()==B.NumGlobalNonzeros(),ierr); EPETRA_TEST_ERR(!A.NumGlobalRows()==B.NumGlobalRows(),ierr); EPETRA_TEST_ERR(!A.NumMyCols()==B.NumMyCols(),ierr); EPETRA_TEST_ERR(!A.NumMyDiagonals()==B.NumMyDiagonals(),ierr); EPETRA_TEST_ERR(!A.NumMyNonzeros()==B.NumMyNonzeros(),ierr); for (int i=0; i<A.NumMyRows(); i++) { int nA, nB; A.NumMyRowEntries(i,nA); B.NumMyRowEntries(i,nB); EPETRA_TEST_ERR(!nA==nB,ierr); } EPETRA_TEST_ERR(!A.NumMyRows()==B.NumMyRows(),ierr); EPETRA_TEST_ERR(!A.OperatorDomainMap().SameAs(B.OperatorDomainMap()),ierr); EPETRA_TEST_ERR(!A.OperatorRangeMap().SameAs(B.OperatorRangeMap()),ierr); EPETRA_TEST_ERR(!A.RowMatrixColMap().SameAs(B.RowMatrixColMap()),ierr); EPETRA_TEST_ERR(!A.RowMatrixRowMap().SameAs(B.RowMatrixRowMap()),ierr); EPETRA_TEST_ERR(!A.UpperTriangular()==B.UpperTriangular(),ierr); EPETRA_TEST_ERR(!A.UseTranspose()==B.UseTranspose(),ierr); int NumVectors = 5; { // No transpose case Epetra_MultiVector X(A.OperatorDomainMap(), NumVectors); Epetra_MultiVector YA1(A.OperatorRangeMap(), NumVectors); Epetra_MultiVector YA2(YA1); Epetra_MultiVector YB1(YA1); Epetra_MultiVector YB2(YA1); X.Random(); bool transA = false; A.SetUseTranspose(transA); B.SetUseTranspose(transA); A.Apply(X,YA1); A.Multiply(transA, X, YA2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2,"A Multiply and A Apply", verbose),ierr); B.Apply(X,YB1); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1,"A Multiply and B Multiply", verbose),ierr); B.Multiply(transA, X, YB2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2,"A Multiply and B Apply", verbose), ierr); } { // transpose case Epetra_MultiVector X(A.OperatorRangeMap(), NumVectors); Epetra_MultiVector YA1(A.OperatorDomainMap(), NumVectors); Epetra_MultiVector YA2(YA1); Epetra_MultiVector YB1(YA1); Epetra_MultiVector YB2(YA1); X.Random(); bool transA = true; A.SetUseTranspose(transA); B.SetUseTranspose(transA); A.Apply(X,YA1); A.Multiply(transA, X, YA2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2, "A Multiply and A Apply (transpose)", verbose),ierr); B.Apply(X,YB1); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1, "A Multiply and B Multiply (transpose)", verbose),ierr); B.Multiply(transA, X,YB2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2, "A Multiply and B Apply (transpose)", verbose),ierr); } Epetra_Vector diagA(A.RowMatrixRowMap()); EPETRA_TEST_ERR(A.ExtractDiagonalCopy(diagA),ierr); Epetra_Vector diagB(B.RowMatrixRowMap()); EPETRA_TEST_ERR(B.ExtractDiagonalCopy(diagB),ierr); EPETRA_TEST_ERR(checkMultiVectors(diagA,diagB, "ExtractDiagonalCopy", verbose),ierr); Epetra_Vector rowA(A.RowMatrixRowMap()); EPETRA_TEST_ERR(A.InvRowSums(rowA),ierr); Epetra_Vector rowB(B.RowMatrixRowMap()); EPETRA_TEST_ERR(B.InvRowSums(rowB),ierr) EPETRA_TEST_ERR(checkMultiVectors(rowA,rowB, "InvRowSums", verbose),ierr); Epetra_Vector colA(A.RowMatrixColMap()); EPETRA_TEST_ERR(A.InvColSums(colA),ierr); Epetra_Vector colB(B.RowMatrixColMap()); EPETRA_TEST_ERR(B.InvColSums(colB),ierr); EPETRA_TEST_ERR(checkMultiVectors(colA,colB, "InvColSums", verbose),ierr); EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf before scaling", verbose), ierr); EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne before scaling", verbose),ierr); EPETRA_TEST_ERR(A.RightScale(colA),ierr); EPETRA_TEST_ERR(B.RightScale(colB),ierr); EPETRA_TEST_ERR(A.LeftScale(rowA),ierr); EPETRA_TEST_ERR(B.LeftScale(rowB),ierr); EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf after scaling", verbose), ierr); EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne after scaling", verbose),ierr); vector<double> valuesA(A.MaxNumEntries()); vector<int> indicesA(A.MaxNumEntries()); vector<double> valuesB(B.MaxNumEntries()); vector<int> indicesB(B.MaxNumEntries()); return(0); for (int i=0; i<A.NumMyRows(); i++) { int nA, nB; EPETRA_TEST_ERR(A.ExtractMyRowCopy(i, A.MaxNumEntries(), nA, &valuesA[0], &indicesA[0]),ierr); EPETRA_TEST_ERR(B.ExtractMyRowCopy(i, B.MaxNumEntries(), nB, &valuesB[0], &indicesB[0]),ierr); EPETRA_TEST_ERR(!nA==nB,ierr); for (int j=0; j<nA; j++) { double curVal = valuesA[j]; int curIndex = indicesA[j]; bool notfound = true; int jj = 0; while (notfound && jj< nB) { if (!checkValues(curVal, valuesB[jj])) notfound = false; jj++; } EPETRA_TEST_ERR(notfound, ierr); vector<int>::iterator p = find(indicesB.begin(),indicesB.end(),curIndex); // find curIndex in indicesB EPETRA_TEST_ERR(p==indicesB.end(), ierr); } } if (verbose) cout << "RowMatrix Methods check OK" << endl; return (ierr); }
//============================================================================= int Amesos_Mumps::ConvertToTriplet(const bool OnlyValues) { Epetra_RowMatrix* ptr; if (Comm().NumProc() == 1) ptr = &Matrix(); else { ptr = &RedistrMatrix(true); } ResetTimer(); #ifdef EXTRA_DEBUG_INFO Epetra_CrsMatrix* Eptr = dynamic_cast<Epetra_CrsMatrix*>( ptr ); if ( ptr->NumGlobalNonzeros() < 300 ) SetICNTL(4,3 ); // Enable more debug info for small matrices if ( ptr->NumGlobalNonzeros() < 42 && Eptr ) { std::cout << " Matrix = " << std::endl ; Eptr->Print( std::cout ) ; } else { assert( Eptr ); } #endif Row.resize(ptr->NumMyNonzeros()); Col.resize(ptr->NumMyNonzeros()); Val.resize(ptr->NumMyNonzeros()); int MaxNumEntries = ptr->MaxNumEntries(); std::vector<int> Indices; std::vector<double> Values; Indices.resize(MaxNumEntries); Values.resize(MaxNumEntries); int count = 0; for (int i = 0; i < ptr->NumMyRows() ; ++i) { int GlobalRow = ptr->RowMatrixRowMap().GID(i); int NumEntries = 0; int ierr; ierr = ptr->ExtractMyRowCopy(i, MaxNumEntries, NumEntries, &Values[0], &Indices[0]); AMESOS_CHK_ERR(ierr); for (int j = 0 ; j < NumEntries ; ++j) { if (OnlyValues == false) { Row[count] = GlobalRow + 1; Col[count] = ptr->RowMatrixColMap().GID(Indices[j]) + 1; } // MS // Added on 15-Mar-05. if (AddToDiag_ && Indices[j] == i) Values[j] += AddToDiag_; Val[count] = Values[j]; count++; } } MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_); assert (count <= ptr->NumMyNonzeros()); return(0); }