//============================================================================= int Amesos_Mumps::SymbolicFactorization() { // erase data if present. if (IsSymbolicFactorizationOK_ && MDS.job != -777) Destroy(); IsSymbolicFactorizationOK_ = false; IsNumericFactorizationOK_ = false; CreateTimer(Comm()); CheckParameters(); AMESOS_CHK_ERR(ConvertToTriplet(false)); #if defined(HAVE_MPI) && defined(HAVE_AMESOS_MPI_C2F) if (MaxProcs_ != Comm().NumProc()) { if(MUMPSComm_) MPI_Comm_free(&MUMPSComm_); std::vector<int> ProcsInGroup(MaxProcs_); for (int i = 0 ; i < MaxProcs_ ; ++i) ProcsInGroup[i] = i; MPI_Group OrigGroup, MumpsGroup; MPI_Comm_group(MPI_COMM_WORLD, &OrigGroup); MPI_Group_incl(OrigGroup, MaxProcs_, &ProcsInGroup[0], &MumpsGroup); MPI_Comm_create(MPI_COMM_WORLD, MumpsGroup, &MUMPSComm_); #ifdef MUMPS_4_9 MDS.comm_fortran = (MUMPS_INT) MPI_Comm_c2f( MUMPSComm_); #else #ifndef HAVE_AMESOS_OLD_MUMPS MDS.comm_fortran = (DMUMPS_INT) MPI_Comm_c2f( MUMPSComm_); #else MDS.comm_fortran = (F_INT) MPI_Comm_c2f( MUMPSComm_); #endif #endif } else { const Epetra_MpiComm* MpiComm = dynamic_cast<const Epetra_MpiComm*>(&Comm()); assert (MpiComm != 0); #ifdef MUMPS_4_9 MDS.comm_fortran = (MUMPS_INT) MPI_Comm_c2f(MpiComm->GetMpiComm()); #else #ifndef HAVE_AMESOS_OLD_MUMPS MDS.comm_fortran = (DMUMPS_INT) MPI_Comm_c2f(MpiComm->GetMpiComm()); #else MDS.comm_fortran = (F_INT) MPI_Comm_c2f(MpiComm->GetMpiComm()); #endif #endif } #else // This next three lines of code were required to make Amesos_Mumps work // with Ifpack_SubdomainFilter. They is usefull in all cases // when using MUMPS on a subdomain. const Epetra_MpiComm* MpiComm = dynamic_cast<const Epetra_MpiComm*>(&Comm()); assert (MpiComm != 0); MDS.comm_fortran = (MUMPS_INT) MPI_Comm_c2f(MpiComm->GetMpiComm()); // only thing I can do, use MPI_COMM_WORLD. This will work in serial as well // Previously, the next line was uncommented, but we don't want MUMPS to work // on the global MPI comm, but on the comm associated with the matrix // MDS.comm_fortran = -987654; #endif MDS.job = -1 ; // Initialization MDS.par = 1 ; // Host IS involved in computations // MDS.sym = MatrixProperty_; MDS.sym = 0; // MatrixProperty_ is unititalized. Furthermore MUMPS // expects only half of the matrix to be provided for // symmetric matrices. Hence setting MDS.sym to be non-zero // indicating that the matrix is symmetric will only work // if we change ConvertToTriplet to pass only half of the // matrix. Bug #2331 and Bug #2332 - low priority RedistrMatrix(true); if (Comm().MyPID() < MaxProcs_) { dmumps_c(&(MDS)); // Initialize MUMPS static_cast<void>( CheckError( ) ); } MDS.n = Matrix().NumGlobalRows(); // fix pointers for nonzero pattern of A. Numerical values // will be entered in PerformNumericalFactorization() if (Comm().NumProc() != 1) { MDS.nz_loc = RedistrMatrix().NumMyNonzeros(); if (Comm().MyPID() < MaxProcs_) { MDS.irn_loc = &Row[0]; MDS.jcn_loc = &Col[0]; } } else { if (Comm().MyPID() == 0) { MDS.nz = Matrix().NumMyNonzeros(); MDS.irn = &Row[0]; MDS.jcn = &Col[0]; } } // scaling if provided by the user if (RowSca_ != 0) { MDS.rowsca = RowSca_; MDS.colsca = ColSca_; } // given ordering if provided by the user if (PermIn_ != 0) { MDS.perm_in = PermIn_; } MDS.job = 1; // Request symbolic factorization SetICNTLandCNTL(); // Perform symbolic factorization ResetTimer(); if (Comm().MyPID() < MaxProcs_) dmumps_c(&(MDS)); SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_); int IntWrong = CheckError()?1:0 ; int AnyWrong; Comm().SumAll( &IntWrong, &AnyWrong, 1 ) ; bool Wrong = AnyWrong > 0 ; if ( Wrong ) { AMESOS_CHK_ERR( StructurallySingularMatrixError ) ; } IsSymbolicFactorizationOK_ = true ; NumSymbolicFact_++; return 0; }
//============================================================================= int Amesos_Paraklete::SymbolicFactorization() { MyPID_ = Comm().MyPID(); NumProcs_ = Comm().NumProc(); IsSymbolicFactorizationOK_ = false; IsNumericFactorizationOK_ = false; #ifdef HAVE_AMESOS_EPETRAEXT transposer_ = static_cast<Teuchos::ENull>( 0 ); #endif CreateTimer(Comm(), 2); ResetTimer(1); // "overhead" time for the following method is considered here AMESOS_CHK_ERR( CreateLocalMatrixAndExporters() ) ; assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() ); SetMaxProcesses(MaxProcesses_, *RowMatrixA_); // // Perform checks in SymbolicFactorization(), but none in // NumericFactorization() or Solve() // assert( ! TrustMe_ ) ; if ( TrustMe_ ) { if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR(10 ); if( UseDataInPlace_ != 1 ) AMESOS_CHK_ERR( 10 ) ; if( Reindex_ ) AMESOS_CHK_ERR( 10 ) ; if( ! Problem_->GetLHS() ) AMESOS_CHK_ERR( 10 ) ; if( ! Problem_->GetRHS() ) AMESOS_CHK_ERR( 10 ) ; if( ! Problem_->GetLHS()->NumVectors() ) AMESOS_CHK_ERR( 10 ) ; if( ! Problem_->GetRHS()->NumVectors() ) AMESOS_CHK_ERR( 10 ) ; SerialB_ = Problem_->GetRHS() ; SerialX_ = Problem_->GetLHS() ; NumVectors_ = SerialX_->NumVectors(); if (MyPID_ == 0) { AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ )); AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ )); } } PrivateParakleteData_->common_ = rcp(new paraklete_common()); const Epetra_MpiComm* MpiComm = dynamic_cast<const Epetra_MpiComm*>(&Comm()); assert (MpiComm != 0); MPI_Comm PK_Comm; // // Create an MPI group with MaxProcesses_ processes // if ( MaxProcesses_ != Comm().NumProc()) { if(ParakleteComm_) { MPI_Comm_free(&ParakleteComm_); ParakleteComm_ = 0 ; } std::vector<int> ProcsInGroup(MaxProcesses_); IamInGroup_ = false; for (int i = 0 ; i < MaxProcesses_ ; ++i) { ProcsInGroup[i] = i; if ( Comm().MyPID() == i ) IamInGroup_ = true; } MPI_Group OrigGroup, ParakleteGroup; MPI_Comm_group(MpiComm->GetMpiComm(), &OrigGroup); MPI_Group_incl(OrigGroup, MaxProcesses_, &ProcsInGroup[0], &ParakleteGroup); MPI_Comm_create(MpiComm->GetMpiComm(), ParakleteGroup, &ParakleteComm_); PK_Comm = ParakleteComm_ ; } else { IamInGroup_ = true; PK_Comm = MpiComm->GetMpiComm() ; } paraklete_common& pk_common = *PrivateParakleteData_->common_ ; cholmod_common *cm = &(pk_common.cm) ; amesos_cholmod_l_start (cm) ; PK_DEBUG_INIT ("pk", cm) ; pk_common.nproc = MaxProcesses_ ; pk_common.myid = Comm().MyPID() ; //pk_common.mpicomm = PK_Comm ; cm->print = 1 ; cm->precise = TRUE ; cm->error_handler = my_handler ; pk_common.tol_diag = 0.001 ; pk_common.tol_offdiag = 0.1 ; pk_common.growth = 2. ; // "overhead" time for the following two methods is considered here AMESOS_CHK_ERR( ExportToSerial() ); AMESOS_CHK_ERR( ConvertToParakleteCRS(true) ); OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // All this time if PARAKLETE time AMESOS_CHK_ERR( PerformSymbolicFactorization() ); NumSymbolicFact_++; IsSymbolicFactorizationOK_ = true; return 0; }