void show_matrix(const char *txt, const Epetra_RowMatrix &matrix, const Epetra_Comm &comm) { int me = comm.MyPID(); if (comm.NumProc() > 10){ if (me == 0){ std::cout << txt << std::endl; std::cout << "Printed matrix format only works for 10 or fewer processes" << std::endl; } return; } int numRows = matrix.NumGlobalRows(); int numCols = matrix.NumGlobalCols(); if ((numRows > 200) || (numCols > 500)){ if (me == 0){ std::cerr << txt << std::endl; std::cerr << "show_matrix: problem is too large to display" << std::endl; } return; } int *myA = new int [numRows * numCols]; make_my_A(matrix, myA, comm); printMatrix(txt, myA, NULL, NULL, numRows, numCols, comm); delete [] myA; }
void show_matrix(const char *txt, const Epetra_LinearProblem &problem, const Epetra_Comm &comm) { int me = comm.MyPID(); if (comm.NumProc() > 10){ if (me == 0){ std::cout << txt << std::endl; std::cout << "Printed matrix format only works for 10 or fewer processes" << std::endl; } return; } Epetra_RowMatrix *matrix = problem.GetMatrix(); Epetra_MultiVector *lhs = problem.GetLHS(); Epetra_MultiVector *rhs = problem.GetRHS(); int numRows = matrix->NumGlobalRows(); int numCols = matrix->NumGlobalCols(); if ((numRows > 200) || (numCols > 500)){ if (me == 0){ std::cerr << txt << std::endl; std::cerr << "show_matrix: problem is too large to display" << std::endl; } return; } int *myA = new int [numRows * numCols]; make_my_A(*matrix, myA, comm); int *myX = new int [numCols]; int *myB = new int [numRows]; memset(myX, 0, sizeof(int) * numCols); memset(myB, 0, sizeof(int) * numRows); const Epetra_BlockMap &lhsMap = lhs->Map(); const Epetra_BlockMap &rhsMap = rhs->Map(); int base = lhsMap.IndexBase(); for (int j=0; j < lhsMap.NumMyElements(); j++){ int colGID = lhsMap.GID(j); myX[colGID - base] = me + 1; } for (int i=0; i < rhsMap.NumMyElements(); i++){ int rowGID = rhsMap.GID(i); myB[rowGID - base] = me + 1; } printMatrix(txt, myA, myX, myB, numRows, numCols, comm); delete [] myA; delete [] myX; delete [] myB; }
static int make_my_A(const Epetra_RowMatrix &matrix, int *myA, const Epetra_Comm &comm) { int me = comm.MyPID(); const Epetra_Map &rowmap = matrix.RowMatrixRowMap(); const Epetra_Map &colmap = matrix.RowMatrixColMap(); int myRows = matrix.NumMyRows(); int numRows = matrix.NumGlobalRows(); int numCols = matrix.NumGlobalCols(); int base = rowmap.IndexBase(); int maxRow = matrix.MaxNumEntries(); memset(myA, 0, sizeof(int) * numRows * numCols); int *myIndices = new int [maxRow]; double *tmp = new double [maxRow]; int rowLen = 0; for (int i=0; i< myRows; i++){ int rc = matrix.ExtractMyRowCopy(i, maxRow, rowLen, tmp, myIndices); if (rc){ if (me == 0){ std::cout << "Error in make_my_A" << std::endl; } return 1; } int *row = myA + (numCols * (rowmap.GID(i) - base)); for (int j=0; j < rowLen; j++){ int colGID = colmap.GID(myIndices[j]); row[colGID - base] = me + 1; } } if (maxRow){ delete [] myIndices; delete [] tmp; } return 0; }
int compute_hypergraph_metrics(const Epetra_RowMatrix &matrix, Isorropia::Epetra::CostDescriber &costs, double &myGoalWeight, double &balance, double &cutn, double &cutl) // output { const Epetra_BlockMap &rmap = static_cast<const Epetra_BlockMap &>(matrix.RowMatrixRowMap()); const Epetra_BlockMap &cmap = static_cast<const Epetra_BlockMap &>(matrix.RowMatrixColMap()); return compute_hypergraph_metrics(rmap, cmap, matrix.NumGlobalCols(), costs, myGoalWeight, balance, cutn, cutl); }
//============================================================================= int Amesos_Dscpack::Solve() { if (IsNumericFactorizationOK_ == false) AMESOS_CHK_ERR(NumericFactorization()); ResetTimer(0); ResetTimer(1); Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix(); if (RowMatrixA == 0) AMESOS_CHK_ERR(-1); // MS // some checks on matrix size if (RowMatrixA->NumGlobalRows() != RowMatrixA->NumGlobalCols()) AMESOS_CHK_ERR(-1); // Convert vector b to a vector in the form that DSCPACK needs it // Epetra_MultiVector* vecX = Problem_->GetLHS(); Epetra_MultiVector* vecB = Problem_->GetRHS(); if ((vecX == 0) || (vecB == 0)) AMESOS_CHK_ERR(-1); // something wrong with input int NumVectors = vecX->NumVectors(); if (NumVectors != vecB->NumVectors()) AMESOS_CHK_ERR(-2); double *dscmapXvalues ; int dscmapXlda ; Epetra_MultiVector dscmapX(DscRowMap(),NumVectors) ; int ierr; AMESOS_CHK_ERR(dscmapX.ExtractView(&dscmapXvalues,&dscmapXlda)); assert (dscmapXlda == NumLocalCols); double *dscmapBvalues ; int dscmapBlda ; Epetra_MultiVector dscmapB(DscRowMap(), NumVectors ) ; ierr = dscmapB.ExtractView( &dscmapBvalues, &dscmapBlda ); AMESOS_CHK_ERR(ierr); assert( dscmapBlda == NumLocalCols ) ; AMESOS_CHK_ERR(dscmapB.Import(*vecB, Importer(), Insert)); VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0); ResetTimer(0); // MS // now solve the problem std::vector<double> ValuesInNewOrder( NumLocalCols ) ; OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); if ( MyDscRank >= 0 ) { for ( int j =0 ; j < NumVectors; j++ ) { for ( int i = 0; i < NumLocalCols; i++ ) { ValuesInNewOrder[i] = dscmapBvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapBlda ] ; } AMESOS_CHK_ERR( DSC_InputRhsLocalVec ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ; AMESOS_CHK_ERR( DSC_Solve ( PrivateDscpackData_->MyDSCObject_ ) ) ; AMESOS_CHK_ERR( DSC_GetLocalSolution ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ; for ( int i = 0; i < NumLocalCols; i++ ) { dscmapXvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapXlda ] = ValuesInNewOrder[i]; } } } SolveTime_ = AddTime("Total solve time", SolveTime_, 0); ResetTimer(0); ResetTimer(1); vecX->Export( dscmapX, Importer(), Insert ) ; VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0); if (ComputeTrueResidual_) ComputeTrueResidual(*(GetProblem()->GetMatrix()), *vecX, *vecB, false, "Amesos_Dscpack"); if (ComputeVectorNorms_) ComputeVectorNorms(*vecX, *vecB, "Amesos_Dscpack"); OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); NumSolve_++; return(0) ; }
//============================================================================= int Amesos_Dscpack::PerformNumericFactorization() { ResetTimer(0); ResetTimer(1); Epetra_RowMatrix* RowMatrixA = Problem_->GetMatrix(); if (RowMatrixA == 0) AMESOS_CHK_ERR(-1); const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ; int numrows = RowMatrixA->NumGlobalRows(); assert( numrows == RowMatrixA->NumGlobalCols() ); // // Call Dscpack to perform Numeric Factorization // std::vector<double> MyANonZ; #if 0 if ( IsNumericFactorizationOK_ ) { DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject); } #endif DscRowMap_ = Teuchos::rcp(new Epetra_Map(numrows, NumLocalCols, LocalStructOldNum, 0, Comm())); if (DscRowMap_.get() == 0) AMESOS_CHK_ERR(-1); Importer_ = rcp(new Epetra_Import(DscRowMap(), OriginalMap)); // // Import from the CrsMatrix // Epetra_CrsMatrix DscMat(Copy, DscRowMap(), 0); AMESOS_CHK_ERR(DscMat.Import(*RowMatrixA, Importer(), Insert)); AMESOS_CHK_ERR(DscMat.FillComplete()); DscColMap_ = Teuchos::rcp(new Epetra_Map(DscMat.RowMatrixColMap())); assert( MyDscRank >= 0 || NumLocalNonz == 0 ) ; assert( MyDscRank >= 0 || NumLocalCols == 0 ) ; assert( MyDscRank >= 0 || NumGlobalCols == 0 ) ; MyANonZ.resize( NumLocalNonz ) ; int NonZIndex = 0 ; int max_num_entries = DscMat.MaxNumEntries() ; std::vector<int> col_indices( max_num_entries ) ; std::vector<double> mat_values( max_num_entries ) ; assert( NumLocalCols == DscRowMap().NumMyElements() ) ; std::vector<int> my_global_elements( NumLocalCols ) ; AMESOS_CHK_ERR(DscRowMap().MyGlobalElements( &my_global_elements[0] ) ) ; std::vector<int> GlobalStructOldColNum( NumGlobalCols ) ; typedef std::pair<int, double> Data; std::vector<Data> sort_array(max_num_entries); std::vector<int> sort_indices(max_num_entries); for ( int i = 0; i < NumLocalCols ; i++ ) { assert( my_global_elements[i] == LocalStructOldNum[i] ) ; int num_entries_this_row; #ifdef USE_LOCAL AMESOS_CHK_ERR( DscMat.ExtractMyRowCopy( i, max_num_entries, num_entries_this_row, &mat_values[0], &col_indices[0] ) ) ; #else AMESOS_CHK_ERR( DscMat.ExtractGlobalRowCopy( DscMat.GRID(i), max_num_entries, num_entries_this_row, &mat_values[0], &col_indices[0] ) ) ; #endif int OldRowNumber = LocalStructOldNum[i] ; if (GlobalStructOwner[ OldRowNumber ] == -1) AMESOS_CHK_ERR(-1); int NewRowNumber = GlobalStructNewColNum[ my_global_elements[ i ] ] ; // // Sort the column elements // for ( int j = 0; j < num_entries_this_row; j++ ) { #ifdef USE_LOCAL sort_array[j].first = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ; sort_indices[j] = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ; #else sort_array[j].first = GlobalStructNewColNum[ col_indices[j] ]; #endif sort_array[j].second = mat_values[j] ; } sort(&sort_array[0], &sort_array[num_entries_this_row]); for ( int j = 0; j < num_entries_this_row; j++ ) { int NewColNumber = sort_array[j].first ; if ( NewRowNumber <= NewColNumber ) MyANonZ[ NonZIndex++ ] = sort_array[j].second ; #ifndef USE_LOCAL assert( NonZIndex <= NumLocalNonz ); // This assert can fail on non-symmetric matrices #endif } } OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); if ( MyDscRank >= 0 ) { const int SchemeCode = 1; #ifndef USE_LOCAL assert( NonZIndex == NumLocalNonz ); #endif AMESOS_CHK_ERR( DSC_NFactor ( PrivateDscpackData_->MyDSCObject_, SchemeCode, &MyANonZ[0], DSC_LLT, DSC_LBLAS3, DSC_DBLAS2 ) ) ; } // if ( MyDscRank >= 0 ) IsNumericFactorizationOK_ = true ; NumFactTime_ = AddTime("Total numeric factorization time", NumFactTime_, 0); return(0); }
int check(Epetra_RowMatrix& A, Epetra_RowMatrix & B, bool verbose) { int ierr = 0; EPETRA_TEST_ERR(!A.Comm().NumProc()==B.Comm().NumProc(),ierr); EPETRA_TEST_ERR(!A.Comm().MyPID()==B.Comm().MyPID(),ierr); EPETRA_TEST_ERR(!A.Filled()==B.Filled(),ierr); EPETRA_TEST_ERR(!A.HasNormInf()==B.HasNormInf(),ierr); EPETRA_TEST_ERR(!A.LowerTriangular()==B.LowerTriangular(),ierr); EPETRA_TEST_ERR(!A.Map().SameAs(B.Map()),ierr); EPETRA_TEST_ERR(!A.MaxNumEntries()==B.MaxNumEntries(),ierr); EPETRA_TEST_ERR(!A.NumGlobalCols()==B.NumGlobalCols(),ierr); EPETRA_TEST_ERR(!A.NumGlobalDiagonals()==B.NumGlobalDiagonals(),ierr); EPETRA_TEST_ERR(!A.NumGlobalNonzeros()==B.NumGlobalNonzeros(),ierr); EPETRA_TEST_ERR(!A.NumGlobalRows()==B.NumGlobalRows(),ierr); EPETRA_TEST_ERR(!A.NumMyCols()==B.NumMyCols(),ierr); EPETRA_TEST_ERR(!A.NumMyDiagonals()==B.NumMyDiagonals(),ierr); EPETRA_TEST_ERR(!A.NumMyNonzeros()==B.NumMyNonzeros(),ierr); for (int i=0; i<A.NumMyRows(); i++) { int nA, nB; A.NumMyRowEntries(i,nA); B.NumMyRowEntries(i,nB); EPETRA_TEST_ERR(!nA==nB,ierr); } EPETRA_TEST_ERR(!A.NumMyRows()==B.NumMyRows(),ierr); EPETRA_TEST_ERR(!A.OperatorDomainMap().SameAs(B.OperatorDomainMap()),ierr); EPETRA_TEST_ERR(!A.OperatorRangeMap().SameAs(B.OperatorRangeMap()),ierr); EPETRA_TEST_ERR(!A.RowMatrixColMap().SameAs(B.RowMatrixColMap()),ierr); EPETRA_TEST_ERR(!A.RowMatrixRowMap().SameAs(B.RowMatrixRowMap()),ierr); EPETRA_TEST_ERR(!A.UpperTriangular()==B.UpperTriangular(),ierr); EPETRA_TEST_ERR(!A.UseTranspose()==B.UseTranspose(),ierr); int NumVectors = 5; { // No transpose case Epetra_MultiVector X(A.OperatorDomainMap(), NumVectors); Epetra_MultiVector YA1(A.OperatorRangeMap(), NumVectors); Epetra_MultiVector YA2(YA1); Epetra_MultiVector YB1(YA1); Epetra_MultiVector YB2(YA1); X.Random(); bool transA = false; A.SetUseTranspose(transA); B.SetUseTranspose(transA); A.Apply(X,YA1); A.Multiply(transA, X, YA2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2,"A Multiply and A Apply", verbose),ierr); B.Apply(X,YB1); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1,"A Multiply and B Multiply", verbose),ierr); B.Multiply(transA, X, YB2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2,"A Multiply and B Apply", verbose), ierr); } { // transpose case Epetra_MultiVector X(A.OperatorRangeMap(), NumVectors); Epetra_MultiVector YA1(A.OperatorDomainMap(), NumVectors); Epetra_MultiVector YA2(YA1); Epetra_MultiVector YB1(YA1); Epetra_MultiVector YB2(YA1); X.Random(); bool transA = true; A.SetUseTranspose(transA); B.SetUseTranspose(transA); A.Apply(X,YA1); A.Multiply(transA, X, YA2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2, "A Multiply and A Apply (transpose)", verbose),ierr); B.Apply(X,YB1); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1, "A Multiply and B Multiply (transpose)", verbose),ierr); B.Multiply(transA, X,YB2); EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2, "A Multiply and B Apply (transpose)", verbose),ierr); } Epetra_Vector diagA(A.RowMatrixRowMap()); EPETRA_TEST_ERR(A.ExtractDiagonalCopy(diagA),ierr); Epetra_Vector diagB(B.RowMatrixRowMap()); EPETRA_TEST_ERR(B.ExtractDiagonalCopy(diagB),ierr); EPETRA_TEST_ERR(checkMultiVectors(diagA,diagB, "ExtractDiagonalCopy", verbose),ierr); Epetra_Vector rowA(A.RowMatrixRowMap()); EPETRA_TEST_ERR(A.InvRowSums(rowA),ierr); Epetra_Vector rowB(B.RowMatrixRowMap()); EPETRA_TEST_ERR(B.InvRowSums(rowB),ierr) EPETRA_TEST_ERR(checkMultiVectors(rowA,rowB, "InvRowSums", verbose),ierr); Epetra_Vector colA(A.RowMatrixColMap()); EPETRA_TEST_ERR(A.InvColSums(colA),ierr); Epetra_Vector colB(B.RowMatrixColMap()); EPETRA_TEST_ERR(B.InvColSums(colB),ierr); EPETRA_TEST_ERR(checkMultiVectors(colA,colB, "InvColSums", verbose),ierr); EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf before scaling", verbose), ierr); EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne before scaling", verbose),ierr); EPETRA_TEST_ERR(A.RightScale(colA),ierr); EPETRA_TEST_ERR(B.RightScale(colB),ierr); EPETRA_TEST_ERR(A.LeftScale(rowA),ierr); EPETRA_TEST_ERR(B.LeftScale(rowB),ierr); EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf after scaling", verbose), ierr); EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne after scaling", verbose),ierr); vector<double> valuesA(A.MaxNumEntries()); vector<int> indicesA(A.MaxNumEntries()); vector<double> valuesB(B.MaxNumEntries()); vector<int> indicesB(B.MaxNumEntries()); return(0); for (int i=0; i<A.NumMyRows(); i++) { int nA, nB; EPETRA_TEST_ERR(A.ExtractMyRowCopy(i, A.MaxNumEntries(), nA, &valuesA[0], &indicesA[0]),ierr); EPETRA_TEST_ERR(B.ExtractMyRowCopy(i, B.MaxNumEntries(), nB, &valuesB[0], &indicesB[0]),ierr); EPETRA_TEST_ERR(!nA==nB,ierr); for (int j=0; j<nA; j++) { double curVal = valuesA[j]; int curIndex = indicesA[j]; bool notfound = true; int jj = 0; while (notfound && jj< nB) { if (!checkValues(curVal, valuesB[jj])) notfound = false; jj++; } EPETRA_TEST_ERR(notfound, ierr); vector<int>::iterator p = find(indicesB.begin(),indicesB.end(),curIndex); // find curIndex in indicesB EPETRA_TEST_ERR(p==indicesB.end(), ierr); } } if (verbose) cout << "RowMatrix Methods check OK" << endl; return (ierr); }
int Amesos_Scalapack::RedistributeA( ) { if( debug_ == 1 ) std::cout << "Entering `RedistributeA()'" << std::endl; Time_->ResetStartTime(); Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); EPETRA_CHK_ERR( RowMatrixA == 0 ) ; const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; int NumberOfProcesses = Comm().NumProc() ; // // Compute a uniform distribution as ScaLAPACK would want it // MyFirstElement - The first element which this processor would have // NumExpectedElemetns - The number of elements which this processor would have // int NumRows_ = RowMatrixA->NumGlobalRows() ; int NumColumns_ = RowMatrixA->NumGlobalCols() ; if ( MaxProcesses_ > 0 ) { NumberOfProcesses = EPETRA_MIN( NumberOfProcesses, MaxProcesses_ ) ; } else { int ProcessNumHeuristic = (1+NumRows_/200)*(1+NumRows_/200); NumberOfProcesses = EPETRA_MIN( NumberOfProcesses, ProcessNumHeuristic ); } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:171" << std::endl; // // Create the ScaLAPACK data distribution. // The TwoD data distribution is created in a completely different // manner and is not transposed (whereas the SaLAPACK 1D data // distribution was transposed) // if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:163" << std::endl; Comm().Barrier(); if ( TwoD_distribution_ ) { if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:166" << std::endl; Comm().Barrier(); npcol_ = EPETRA_MIN( NumberOfProcesses, EPETRA_MAX ( 2, (int) sqrt( NumberOfProcesses * 0.5 ) ) ) ; nprow_ = NumberOfProcesses / npcol_ ; // // Create the map for FatA - our first intermediate matrix // int NumMyElements = RowMatrixA->RowMatrixRowMap().NumMyElements() ; std::vector<int> MyGlobalElements( NumMyElements ); RowMatrixA->RowMatrixRowMap().MyGlobalElements( &MyGlobalElements[0] ) ; int NumMyColumns = RowMatrixA->RowMatrixColMap().NumMyElements() ; std::vector<int> MyGlobalColumns( NumMyColumns ); RowMatrixA->RowMatrixColMap().MyGlobalElements( &MyGlobalColumns[0] ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:194" << std::endl; std::vector<int> MyFatElements( NumMyElements * npcol_ ); for( int LocalRow=0; LocalRow<NumMyElements; LocalRow++ ) { for (int i = 0 ; i < npcol_; i++ ){ MyFatElements[LocalRow*npcol_+i] = MyGlobalElements[LocalRow]*npcol_+i; } } Epetra_Map FatInMap( npcol_*NumRows_, NumMyElements*npcol_, &MyFatElements[0], 0, Comm() ); // // Create FatIn, our first intermediate matrix // Epetra_CrsMatrix FatIn( Copy, FatInMap, 0 ); std::vector<std::vector<int> > FatColumnIndices(npcol_,std::vector<int>(1)); std::vector<std::vector<double> > FatMatrixValues(npcol_,std::vector<double>(1)); std::vector<int> FatRowPtrs(npcol_); // A FatRowPtrs[i] = the number // of entries in local row LocalRow*npcol_ + i if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:219" << std::endl; // mypcol_ = iam_%npcol_; myprow_ = (iam_/npcol_)%nprow_; if ( iam_ >= nprow_ * npcol_ ) { myprow_ = nprow_; mypcol_ = npcol_; } // Each row is split into npcol_ rows, with each of the // new rows containing only those elements belonging to // its process column (in the ScaLAPACK 2D process grid) // int MaxNumIndices = RowMatrixA->MaxNumEntries(); int NumIndices; std::vector<int> ColumnIndices(MaxNumIndices); std::vector<double> MatrixValues(MaxNumIndices); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:232 NumMyElements = " << NumMyElements << std::endl; nb_ = grid_nb_; for( int LocalRow=0; LocalRow<NumMyElements; ++LocalRow ) { RowMatrixA->ExtractMyRowCopy( LocalRow, MaxNumIndices, NumIndices, &MatrixValues[0], &ColumnIndices[0] ); for (int i=0; i<npcol_; i++ ) FatRowPtrs[i] = 0 ; // // Deal the individual matrix entries out to the row owned by // the process to which this matrix entry will belong. // for( int i=0 ; i<NumIndices ; ++i ) { int GlobalCol = MyGlobalColumns[ ColumnIndices[i] ]; int pcol_i = pcolnum( GlobalCol, nb_, npcol_ ) ; if ( FatRowPtrs[ pcol_i ]+1 >= FatColumnIndices[ pcol_i ].size() ) { FatColumnIndices[ pcol_i ]. resize( 2 * FatRowPtrs[ pcol_i ]+1 ); FatMatrixValues[ pcol_i ]. resize( 2 * FatRowPtrs[ pcol_i ]+1 ); } FatColumnIndices[pcol_i][FatRowPtrs[pcol_i]] = GlobalCol ; FatMatrixValues[pcol_i][FatRowPtrs[pcol_i]] = MatrixValues[i]; FatRowPtrs[ pcol_i ]++; } // // Insert each of the npcol_ rows individually // for ( int pcol_i = 0 ; pcol_i < npcol_ ; pcol_i++ ) { FatIn.InsertGlobalValues( MyGlobalElements[LocalRow]*npcol_ + pcol_i, FatRowPtrs[ pcol_i ], &FatMatrixValues[ pcol_i ][0], &FatColumnIndices[ pcol_i ][0] ); } } FatIn.FillComplete( false ); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:260" << std::endl; if ( debug_ == 1) std::cout << "Amesos_Scalapack.cpp:265B" << " iam_ = " << iam_ << " nb_ = " << nb_ << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << std::endl; // // Compute the map for our second intermediate matrix, FatOut // // Compute directly int UniformRows = ( NumRows_ / ( nprow_ * nb_ ) ) * nb_ ; int AllExcessRows = NumRows_ - UniformRows * nprow_ ; int OurExcessRows = EPETRA_MIN( nb_, AllExcessRows - ( myprow_ * nb_ ) ) ; OurExcessRows = EPETRA_MAX( 0, OurExcessRows ); NumOurRows_ = UniformRows + OurExcessRows ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:277" << std::endl; int UniformColumns = ( NumColumns_ / ( npcol_ * nb_ ) ) * nb_ ; int AllExcessColumns = NumColumns_ - UniformColumns * npcol_ ; int OurExcessColumns = EPETRA_MIN( nb_, AllExcessColumns - ( mypcol_ * nb_ ) ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:281" << std::endl; OurExcessColumns = EPETRA_MAX( 0, OurExcessColumns ); NumOurColumns_ = UniformColumns + OurExcessColumns ; if ( iam_ >= nprow_ * npcol_ ) { UniformRows = 0; NumOurRows_ = 0; NumOurColumns_ = 0; } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:295" << std::endl; #if 0 // Compute using ScaLAPACK's numroc routine, assert agreement int izero = 0; // All matrices start at process 0 int NumRocSays = numroc_( &NumRows_, &nb_, &myprow_, &izero, &nprow_ ); assert( NumOurRows_ == NumRocSays ); #endif // // Compute the rows which this process row owns in the ScaLAPACK 2D // process grid. // std::vector<int> AllOurRows(NumOurRows_); int RowIndex = 0 ; int BlockRow = 0 ; for ( ; BlockRow < UniformRows / nb_ ; BlockRow++ ) { for ( int RowOffset = 0; RowOffset < nb_ ; RowOffset++ ) { AllOurRows[RowIndex++] = BlockRow*nb_*nprow_ + myprow_*nb_ + RowOffset ; } } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:315" << std::endl; assert ( BlockRow == UniformRows / nb_ ) ; for ( int RowOffset = 0; RowOffset < OurExcessRows ; RowOffset++ ) { AllOurRows[RowIndex++] = BlockRow*nb_*nprow_ + myprow_*nb_ + RowOffset ; } assert( RowIndex == NumOurRows_ ); // // Distribute those rows amongst all the processes in that process row // This is an artificial distribution with the following properties: // 1) It is a 1D data distribution (each row belogs entirely to // a single process // 2) All data which will eventually belong to a given process row, // is entirely contained within the processes in that row. // Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:312" << std::endl; // // Compute MyRows directly // std::vector<int>MyRows(NumOurRows_); RowIndex = 0 ; BlockRow = 0 ; for ( ; BlockRow < UniformRows / nb_ ; BlockRow++ ) { for ( int RowOffset = 0; RowOffset < nb_ ; RowOffset++ ) { MyRows[RowIndex++] = BlockRow*nb_*nprow_*npcol_ + myprow_*nb_*npcol_ + RowOffset*npcol_ + mypcol_ ; } } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:326" << std::endl; assert ( BlockRow == UniformRows / nb_ ) ; for ( int RowOffset = 0; RowOffset < OurExcessRows ; RowOffset++ ) { MyRows[RowIndex++] = BlockRow*nb_*nprow_*npcol_ + myprow_*nb_*npcol_ + RowOffset*npcol_ + mypcol_ ; } Comm().Barrier(); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:334" << std::endl; Comm().Barrier(); for (int i=0; i < NumOurRows_; i++ ) { assert( MyRows[i] == AllOurRows[i]*npcol_+mypcol_ ); } Comm().Barrier(); if ( debug_ == 1) std::cout << "Amesos_Scalapack.cpp:340" << " iam_ = " << iam_ << " myprow_ = " << myprow_ << " mypcol_ = " << mypcol_ << " NumRows_ = " << NumRows_ << " NumOurRows_ = " << NumOurRows_ << std::endl; Comm().Barrier(); Epetra_Map FatOutMap( npcol_*NumRows_, NumOurRows_, &MyRows[0], 0, Comm() ); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:344" << std::endl; Comm().Barrier(); if ( FatOut_ ) delete FatOut_ ; FatOut_ = new Epetra_CrsMatrix( Copy, FatOutMap, 0 ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:348" << std::endl; Epetra_Export ExportToFatOut( FatInMap, FatOutMap ) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:360" << std::endl; FatOut_->Export( FatIn, ExportToFatOut, Add ); FatOut_->FillComplete( false ); // // Create a map to allow us to redistribute the vectors X and B // Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator()); const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap() ; assert( NumGlobalElements_ == OriginalMap.NumGlobalElements() ) ; int NumMyVecElements = 0 ; if ( mypcol_ == 0 ) { NumMyVecElements = NumOurRows_; } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:385" << std::endl; if (VectorMap_) { delete VectorMap_ ; VectorMap_ = 0 ; } VectorMap_ = new Epetra_Map( NumGlobalElements_, NumMyVecElements, &AllOurRows[0], 0, Comm() ); if ( debug_ == 1) std::cout << "iam_ = " << iam_ << " Amesos_Scalapack.cpp:393 debug_ = " << debug_ << std::endl; } else { nprow_ = 1 ; npcol_ = NumberOfProcesses / nprow_ ; assert ( nprow_ * npcol_ == NumberOfProcesses ) ; m_per_p_ = ( NumRows_ + NumberOfProcesses - 1 ) / NumberOfProcesses ; int MyFirstElement = EPETRA_MIN( iam_ * m_per_p_, NumRows_ ) ; int MyFirstNonElement = EPETRA_MIN( (iam_+1) * m_per_p_, NumRows_ ) ; int NumExpectedElements = MyFirstNonElement - MyFirstElement ; assert( NumRows_ == RowMatrixA->NumGlobalRows() ) ; if ( ScaLAPACK1DMap_ ) delete( ScaLAPACK1DMap_ ) ; ScaLAPACK1DMap_ = new Epetra_Map( NumRows_, NumExpectedElements, 0, Comm() ); if ( ScaLAPACK1DMatrix_ ) delete( ScaLAPACK1DMatrix_ ) ; ScaLAPACK1DMatrix_ = new Epetra_CrsMatrix(Copy, *ScaLAPACK1DMap_, 0); Epetra_Export ExportToScaLAPACK1D_( OriginalMap, *ScaLAPACK1DMap_); ScaLAPACK1DMatrix_->Export( *RowMatrixA, ExportToScaLAPACK1D_, Add ); ScaLAPACK1DMatrix_->FillComplete( false ) ; } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << " Amesos_Scalapack.cpp:417 debug_ = " << debug_ << std::endl; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:402" << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << std::endl ; int info; const int zero = 0 ; if ( ictxt_ == -1313 ) { ictxt_ = 0 ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:408" << std::endl; SL_INIT_F77(&ictxt_, &nprow_, &npcol_) ; } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:410A" << std::endl; int nprow; int npcol; int myrow; int mycol; BLACS_GRIDINFO_F77(&ictxt_, &nprow, &npcol, &myrow, &mycol) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "iam_ = " << iam_ << " Amesos_Scalapack.cpp:410" << std::endl; if ( iam_ < nprow_ * npcol_ ) { assert( nprow == nprow_ ) ; if ( npcol != npcol_ ) std::cout << "Amesos_Scalapack.cpp:430 npcol = " << npcol << " npcol_ = " << npcol_ << std::endl ; assert( npcol == npcol_ ) ; if ( TwoD_distribution_ ) { assert( myrow == myprow_ ) ; assert( mycol == mypcol_ ) ; lda_ = EPETRA_MAX(1,NumOurRows_) ; } else { assert( myrow == 0 ) ; assert( mycol == iam_ ) ; nb_ = m_per_p_; lda_ = EPETRA_MAX(1,NumGlobalElements_); } if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp: " << __LINE__ << " TwoD_distribution_ = " << TwoD_distribution_ << " NumGlobalElements_ = " << NumGlobalElements_ << " debug_ = " << debug_ << " nb_ = " << nb_ << " lda_ = " << lda_ << " nprow_ = " << nprow_ << " npcol_ = " << npcol_ << " myprow_ = " << myprow_ << " mypcol_ = " << mypcol_ << " iam_ = " << iam_ << std::endl ; AMESOS_PRINT( myprow_ ); DESCINIT_F77(DescA_, &NumGlobalElements_, &NumGlobalElements_, &nb_, &nb_, &zero, &zero, &ictxt_, &lda_, &info) ; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:441" << std::endl; assert( info == 0 ) ; } else { DescA_[0] = -13; if ( debug_ == 1) std::cout << "iam_ = " << iam_ << "Amesos_Scalapack.cpp:458 nprow = " << nprow << std::endl; assert( nprow == -1 ) ; } if ( debug_ == 1) std::cout << "Amesos_Scalapack.cpp:446" << std::endl; MatTime_ += Time_->ElapsedTime(); return 0; }