// ================================================ ====== ==== ==== == = // Computes C= <me> * A int ML_Epetra::ML_RefMaxwell_11_Operator::MatrixMatrix_Multiply(const Epetra_CrsMatrix & A, ML_Comm *comm, ML_Operator **C) const { ML_Operator *SM_ML,*A_ML,*temp1,*temp2; /* General Stuff */ ML_Comm* temp = global_comm; A_ML = ML_Operator_Create(comm); *C = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)&A,A_ML); /* Do the SM part */ SM_ML = ML_Operator_Create(comm); temp1 = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)SM_Matrix_,SM_ML); ML_2matmult(SM_ML,A_ML,temp1,ML_CSR_MATRIX); ML_Matrix_Print(temp1,*Comm_,*RangeMap_,"smp.dat"); /* Do the Addon part */ Addon_->MatrixMatrix_Multiply(A,comm,&temp2); ML_Matrix_Print(temp2,*Comm_,*RangeMap_,"add_p.dat"); /* Add the matrices together */ ML_Operator_Add(temp2,temp1,*C,ML_CSR_MATRIX,1.0); ML_Matrix_Print(*C,*Comm_,*RangeMap_,"tfinal.dat"); /* Cleanup */ global_comm = temp; ML_Operator_Destroy(&A_ML); ML_Operator_Destroy(&SM_ML); ML_Operator_Destroy(&temp1); ML_Operator_Destroy(&temp2); return 0; }/*end MatrixMatrix_Multiply*/
// ================================================ ====== ==== ==== == = // Forms the coarse matrix, given the prolongator int ML_Epetra::FaceMatrixFreePreconditioner::FormCoarseMatrix() { CoarseMat_ML = ML_Operator_Create(ml_comm_); CoarseMat_ML->data_destroy=free; ML_Operator *Temp_ML=0; ML_Operator *R= ML_Operator_Create(ml_comm_); ML_Operator *P= ML_Operator_Create(ml_comm_); /* Build ML_Operator version of Prolongator_, Restriction Operator */ ML_CHK_ERR(ML_Operator_WrapEpetraCrsMatrix(Prolongator_,P,verbose_)); P->num_rigid=P->num_PDEs=dim; //NTS: ML_CHK_ERR won't work on this: it returns 1 ML_Operator_Transpose_byrow(P, R); /* OPTION: Disable the addon */ const Epetra_CrsMatrix *Op11crs = dynamic_cast<const Epetra_CrsMatrix*>(&*Operator_); const Epetra_Operator_With_MatMat *Op11mm = dynamic_cast<const Epetra_Operator_With_MatMat*>(&*Operator_); /* Do the A*P with or without addon*/ if(Op11crs){ if(verbose_ && !Comm_->MyPID()) printf("FMFP: Running *without* addon\n"); ML_Operator *SM_ML = ML_Operator_Create(ml_comm_); Temp_ML = ML_Operator_Create(ml_comm_); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Op11crs,SM_ML,verbose_); ML_2matmult(SM_ML,P,Temp_ML,ML_CSR_MATRIX); ML_Operator_Destroy(&SM_ML); } else if(Op11mm){ if(verbose_ && !Comm_->MyPID()) printf("FMFP: Running with addon\n"); ML_CHK_ERR(Op11mm->MatrixMatrix_Multiply(*Prolongator_,ml_comm_,&Temp_ML)); } else{ if(!Comm_->MyPID()) printf("ERROR: FMFP Illegal Operator\n"); delete R; ML_CHK_ERR(-1); } /* Do R * AP */ R->num_rigid=R->num_PDEs=dim; ML_2matmult_block(R, Temp_ML,CoarseMat_ML,ML_CSR_MATRIX); /* Wrap to Epetra-land */ int nnz=100; double time; ML_Operator2EpetraCrsMatrix(CoarseMat_ML,CoarseMatrix,nnz,true,time,0,verbose_); // NTS: This is a hack to get around the sticking ones on the diagonal issue; /* Cleanup */ ML_Operator_Destroy(&P); ML_Operator_Destroy(&R); ML_Operator_Destroy(&Temp_ML); ML_Operator_Destroy(&CoarseMat_ML);CoarseMat_ML=0;//HAX return 0; }/*end FormCoarseMatrix*/
// ================================================ ====== ==== ==== == = // Computes C= <me> * A int ML_Epetra::Epetra_Multi_CrsMatrix::MatrixMatrix_Multiply(const Epetra_CrsMatrix & A, ML_Comm *comm, ML_Operator **C) const { int rv=0; ML_Comm* temp = global_comm; /* Setup for 1st Matmat */ ML_Operator * MV[2]={0,0},*CV; MV[(NumMatrices_-1)%2]= ML_Operator_Create(comm); rv=ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)&A,MV[(NumMatrices_-1)%2]); ML_CHK_ERR(rv); /* Do the matmats */ for(int i=NumMatrices_-1;rv==0 && i>=0;i--){ /* Do pre-wraps */ if(MV[(i+1)%2] && i!=NumMatrices_-1) ML_Operator_Destroy(&MV[(i+1)%2]); MV[(i+1)%2]=ML_Operator_Create(comm); CV=ML_Operator_Create(comm); rv=ML_Operator_WrapEpetraCrsMatrix(CrsMatrices_[i],CV); ML_CHK_ERR(rv); /* Do matmat */ ML_2matmult(CV,MV[i%2],MV[(i+1)%2],ML_CSR_MATRIX); ML_Operator_Destroy(&CV); }/*end for*/ global_comm = temp; /* Final Postwrap */ *C=MV[1]; /* Cleanup */ if(MV[0]) ML_Operator_Destroy(&MV[0]); return rv; }/*end MatrixMatrix_Multiply*/
// ================================================ ====== ==== ==== == = //! Build the face-to-node prolongator described by Bochev, Siefert, Tuminaro, Xu and Zhu (2007). int ML_Epetra::FaceMatrixFreePreconditioner::PBuildSparsity(ML_Operator *P, Epetra_CrsMatrix *&Psparse){ /* Create wrapper to do abs(T) */ // NTS: Assume D0 has already been reindexed by now. ML_Operator* AbsFN_ML = ML_Operator_Create(ml_comm_); ML_CHK_ERR(ML_Operator_WrapEpetraCrsMatrix(const_cast<Epetra_CrsMatrix*>(&*FaceNode_Matrix_),AbsFN_ML,verbose_)); ML_Operator_Set_Getrow(AbsFN_ML,AbsFN_ML->outvec_leng,CSR_getrow_ones); /* Form abs(T) * P_n */ ML_Operator* AbsFNP = ML_Operator_Create(ml_comm_); ML_2matmult(AbsFN_ML,P,AbsFNP, ML_CSR_MATRIX); /* Wrap P_n into Epetra-land */ Epetra_CrsMatrix_Wrap_ML_Operator(AbsFNP,*Comm_,*FaceRangeMap_,&Psparse,Copy,0); /* Nuke the rows in Psparse */ if(BCfaces_.size()>0) Apply_BCsToMatrixRows(BCfaces_.get(),BCfaces_.size(),*Psparse); // Cleanup ML_Operator_Destroy(&AbsFN_ML); ML_Operator_Destroy(&AbsFNP); return 0; }
// ================================================ ====== ==== ==== == = // Computes C= A^T * <me> * A. OptimizeStorage *must* be called for both A and the // matrices in *this, before this routine can work. int ML_Epetra::ML_RefMaxwell_11_Operator::PtAP(const Epetra_CrsMatrix & P, ML_Comm *comm, ML_Operator **C) const{ ML_Operator *SM_ML,*P_ML,*R_ML,*PtSMP_ML,*temp1,*temp2,*opwrap,*D0_M1_P_ML; /* General Stuff */ ML_Comm* temp = global_comm; P_ML = ML_Operator_Create(comm); R_ML = ML_Operator_Create(comm);; ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)&P,P_ML); ML_Operator_Transpose_byrow(P_ML,R_ML); /* Do the SM part */ SM_ML = ML_Operator_Create(comm); temp1 = ML_Operator_Create(comm); PtSMP_ML = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)SM_Matrix_,SM_ML); ML_2matmult(SM_ML,P_ML,temp1,ML_CSR_MATRIX); ML_2matmult_block(R_ML,temp1,PtSMP_ML,ML_CSR_MATRIX); ML_Operator_Destroy(&temp1); ML_Operator_Destroy(&SM_ML); #ifdef MANUALLY_TRANSPOSE_D0 ML_Operator_Destroy(&R_ML); #endif ML_Matrix_Print(PtSMP_ML,*Comm_,*RangeMap_,"ptsmp.dat"); #ifdef MANUALLY_TRANSPOSE_D0 /* Do the Addon: Step #1: M1 * P*/ opwrap = ML_Operator_Create(comm); temp1 = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Addon_Matrix_[4],opwrap); ML_2matmult(opwrap,P_ML,temp1,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); ML_Operator_Destroy(&P_ML); /* Do the Addon: Step #2: D0^T *(M1 * P)*/ opwrap = ML_Operator_Create(comm); D0_M1_P_ML = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Addon_Matrix_[3],opwrap); ML_2matmult(opwrap,temp1,D0_M1_P_ML,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); ML_Operator_Destroy(&temp1); /* Do the Addon: Step #3: M0^{-1} * (D0^T * M1 * P)*/ opwrap = ML_Operator_Create(comm); temp1 = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Addon_Matrix_[2],opwrap); ML_2matmult(opwrap,D0_M1_P_ML,temp1,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); /* Do the Addon: Step #4: Transpose (D0^T * M1 * P) & multiply by output from Step 3*/ opwrap = ML_Operator_Create(comm); temp2 = ML_Operator_Create(comm); ML_Operator_Transpose_byrow(D0_M1_P_ML,opwrap); ML_2matmult(opwrap,temp1,temp2,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); ML_Operator_Destroy(&temp1); ML_Operator_Destroy(&D0_M1_P_ML); ML_Matrix_Print(temp2,*Comm_,*RangeMap_,"pt_add_p.dat"); #else ML_Operator *P_M1_D0_ML; /* Do the Addon: Step #1: P^T * M1 */ opwrap = ML_Operator_Create(comm); temp1 = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Addon_Matrix_[0],opwrap); ML_2matmult_block(R_ML,opwrap,temp1,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); ML_Operator_Destroy(&P_ML); ML_Operator_Destroy(&R_ML); /* Do the Addon: Step #2: (P^T * M1) * D0*/ opwrap = ML_Operator_Create(comm); P_M1_D0_ML = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Addon_Matrix_[1],opwrap); ML_2matmult_block(temp1,opwrap,P_M1_D0_ML,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); ML_Operator_Destroy(&temp1); /* Do the Addon: Step #3: (P^T * M1 * D0) * M0^{-1} */ opwrap = ML_Operator_Create(comm); temp1 = ML_Operator_Create(comm); ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Addon_Matrix_[2],opwrap); ML_2matmult(P_M1_D0_ML,opwrap,temp1,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); /* Do the Addon: Step #4: Transpose (P^T * M1 * D0) & multiply by output from Step 3*/ opwrap = ML_Operator_Create(comm); temp2 = ML_Operator_Create(comm); ML_Operator_Transpose_byrow(P_M1_D0_ML,opwrap); ML_2matmult(temp1,opwrap,temp2,ML_CSR_MATRIX); ML_Operator_Destroy(&opwrap); ML_Operator_Destroy(&temp1); ML_Operator_Destroy(&P_M1_D0_ML); ML_Matrix_Print(temp2,*Comm_,*RangeMap_,"pt_add_p_rev.dat"); #endif /* Add the matrices together */ ML_Operator_Add(PtSMP_ML,temp2,*C,ML_CSR_MATRIX,1.0); ML_Matrix_Print(*C,*Comm_,*RangeMap_,"ptap.dat"); /* Cleanup */ global_comm = temp; ML_Operator_Destroy(&temp2); ML_Operator_Destroy(&PtSMP_ML); return 0; }
// ================================================ ====== ==== ==== == = //! Build the face-to-node prolongator described by Bochev, Siefert, Tuminaro, Xu and Zhu (2007). int ML_Epetra::FaceMatrixFreePreconditioner::BuildProlongator() { /* Wrap TMT_Matrix in a ML_Operator */ ML_Operator* TMT_ML = ML_Operator_Create(ml_comm_); ML_Operator_WrapEpetraCrsMatrix(const_cast<Epetra_CrsMatrix*>(&*TMT_Matrix_),TMT_ML); /* Nodal Aggregation */ ML_Aggregate_Struct *MLAggr=0; ML_Operator *P=0; int NumAggregates; int rv=ML_Epetra::RefMaxwell_Aggregate_Nodes(*TMT_Matrix_,List_,ml_comm_,std::string("FMFP (level 0) :"),MLAggr,P,NumAggregates); if(rv || !P) {if(!Comm_->MyPID()) printf("ERROR: Building nodal P\n");ML_CHK_ERR(-1);} /* Build 1-unknown sparsity of prolongator */ Epetra_CrsMatrix *Psparse=0; PBuildSparsity(P,Psparse); if(!Psparse) {if(!Comm_->MyPID()) printf("ERROR: Building Psparse\n");ML_CHK_ERR(-2);} /* Build the "nullspace" */ Epetra_MultiVector *nullspace; BuildNullspace(nullspace); if(!nullspace) {if(!Comm_->MyPID()) printf("ERROR: Building Nullspace\n");ML_CHK_ERR(-3);} /* Build the DomainMap of the new operator*/ const Epetra_Map & FineColMap = Psparse->ColMap(); CoarseMap_=new Epetra_Map(-1,NumAggregates*dim,0,*Comm_); /* Allocate the Prolongator_ */ int max_nz_per_row=Psparse->MaxNumEntries(); Prolongator_=new Epetra_CrsMatrix(Copy,*FaceRangeMap_,0); int ne1, *idx1, *idx2; idx2=new int [dim*max_nz_per_row]; double *vals1, *vals2; vals2=new double[dim*max_nz_per_row]; int nonzeros; for(int i=0;i<Prolongator_->NumMyRows();i++){ Psparse->ExtractMyRowView(i,ne1,vals1,idx1); nonzeros=0; for(int j=0;j<ne1;j++) nonzeros+=ABS(vals1[j])>0; for(int j=0;j<ne1;j++){ for(int k=0;k<dim;k++) { idx2[j*dim+k]=FineColMap.GID(idx1[j])*dim+k; //FIX: This works only because there's an implicit linear mapping which //we're exploiting. if(idx2[j*dim+k]==-1) printf("[%d] ERROR: idx1[j]=%d / idx1[j]*dim+k=%d does not have a GID!\n",Comm_->MyPID(),idx1[j],idx1[j]*dim+k); if(vals1[j]==0 ) vals2[j*dim+k]=0; else vals2[j*dim+k]=(*nullspace)[k][i] / nonzeros; }/*end for*/ }/*end for*/ Prolongator_->InsertGlobalValues(FaceRangeMap_->GID(i),dim*ne1,vals2,idx2); }/*end for*/ /* FillComplete / OptimizeStorage for Prolongator*/ Prolongator_->FillComplete(*CoarseMap_,*FaceRangeMap_); Prolongator_->OptimizeStorage(); #ifndef NO_OUTPUT /* DEBUG: Dump aggregates */ Epetra_IntVector AGG(View,*NodeDomainMap_,MLAggr->aggr_info[0]); IVOUT(AGG,"agg.dat"); EpetraExt::RowMatrixToMatlabFile("fmfp_psparse.dat",*Psparse); EpetraExt::RowMatrixToMatlabFile("fmfp_prolongator.dat",*Prolongator_); EpetraExt::VectorToMatrixMarketFile("fmfp_null0.dat",*(*nullspace)(0)); EpetraExt::VectorToMatrixMarketFile("fmfp_null1.dat",*(*nullspace)(1)); EpetraExt::VectorToMatrixMarketFile("fmfp_null2.dat",*(*nullspace)(2)); #endif /* EXPERIMENTAL: Normalize Prolongator Columns */ bool normalize_prolongator=List_.get("face matrix free: normalize prolongator",false); if(normalize_prolongator){ Epetra_Vector n_vector(*CoarseMap_,false); Prolongator_->InvColSums(n_vector); Prolongator_->RightScale(n_vector); }/*end if*/ /* Post-wrapping to convert to ML indexing */ #ifdef HAVE_ML_EPETRAEXT Prolongator_ = dynamic_cast<Epetra_CrsMatrix*>(ModifyEpetraMatrixColMap(*Prolongator_,ProlongatorColMapTrans_,"Prolongator",(verbose_&&!Comm_->MyPID()))); #endif /* Cleanup */ ML_qr_fix_Destroy(); ML_Aggregate_Destroy(&MLAggr); ML_Operator_Destroy(&TMT_ML); ML_Operator_Destroy(&P); delete nullspace; delete Psparse; delete [] idx2; delete [] vals2; return 0; }/*end BuildProlongator_*/
// ================================================ ====== ==== ==== == = int ML_Epetra::RefMaxwell_Aggregate_Nodes(const Epetra_CrsMatrix & A, Teuchos::ParameterList & List, ML_Comm * ml_comm, std::string PrintMsg, ML_Aggregate_Struct *& MLAggr,ML_Operator *&P, int &NumAggregates){ /* Output level */ bool verbose, very_verbose; int OutputLevel = List.get("ML output", -47); if(OutputLevel == -47) OutputLevel = List.get("output", 1); if(OutputLevel>=15) very_verbose=verbose=true; if(OutputLevel > 5) {very_verbose=false;verbose=true;} else very_verbose=verbose=false; /* Wrap A in a ML_Operator */ ML_Operator* A_ML = ML_Operator_Create(ml_comm); ML_Operator_WrapEpetraCrsMatrix(const_cast<Epetra_CrsMatrix*>(&A),A_ML); /* Pull Teuchos Options */ std::string CoarsenType = List.get("aggregation: type", "Uncoupled"); double Threshold = List.get("aggregation: threshold", 0.0); int NodesPerAggr = List.get("aggregation: nodes per aggregate", ML_Aggregate_Get_OptimalNumberOfNodesPerAggregate()); bool UseAux = List.get("aggregation: aux: enable",false); double AuxThreshold = List.get("aggregation: aux: threshold",0.0); int MaxAuxLevels = List.get("aggregation: aux: max levels",10); ML_Aggregate_Create(&MLAggr); ML_Aggregate_Set_MaxLevels(MLAggr, 2); ML_Aggregate_Set_StartLevel(MLAggr, 0); ML_Aggregate_Set_Threshold(MLAggr, Threshold); ML_Aggregate_Set_MaxCoarseSize(MLAggr,1); MLAggr->cur_level = 0; ML_Aggregate_Set_Reuse(MLAggr); MLAggr->keep_agg_information = 1; P = ML_Operator_Create(ml_comm); /* Process Teuchos Options */ if (CoarsenType == "Uncoupled") ML_Aggregate_Set_CoarsenScheme_Uncoupled(MLAggr); else if (CoarsenType == "Uncoupled-MIS"){ ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr); } else if (CoarsenType == "METIS"){ ML_Aggregate_Set_CoarsenScheme_METIS(MLAggr); ML_Aggregate_Set_NodesPerAggr(0, MLAggr, 0, NodesPerAggr); }/*end if*/ else { if(!A.Comm().MyPID()) printf("%s Unsupported (1,1) block aggregation type(%s), resetting to uncoupled-mis\n",PrintMsg.c_str(),CoarsenType.c_str()); ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr); } /* Setup Aux Data */ if(UseAux) { A_ML->aux_data->enable=1; A_ML->aux_data->threshold=AuxThreshold; A_ML->aux_data->max_level=MaxAuxLevels; ML_Init_Aux(A_ML,List); if(verbose && !A.Comm().MyPID()) { printf("%s Using auxiliary matrix\n",PrintMsg.c_str()); printf("%s aux threshold = %e\n",PrintMsg.c_str(),A_ML->aux_data->threshold); } } /* Aggregate Nodes */ int printlevel=ML_Get_PrintLevel(); if(verbose) ML_Set_PrintLevel(10); NumAggregates = ML_Aggregate_Coarsen(MLAggr,A_ML, &P, ml_comm); if(verbose) ML_Set_PrintLevel(printlevel); if (NumAggregates == 0){ std::cerr << "Found 0 aggregates, perhaps the problem is too small." << std::endl; ML_CHK_ERR(-2); }/*end if*/ else if(very_verbose) printf("[%d] %s %d aggregates created invec_leng=%d\n",A.Comm().MyPID(),PrintMsg.c_str(),NumAggregates,P->invec_leng); if(verbose){ int globalAggs=0; A.Comm().SumAll(&NumAggregates,&globalAggs,1); if(!A.Comm().MyPID()) { printf("%s Aggregation threshold = %e\n",PrintMsg.c_str(),Threshold); printf("%s Global aggregates = %d\n",PrintMsg.c_str(),globalAggs); } } /* Cleanup */ ML_qr_fix_Destroy(); if(UseAux) ML_Finalize_Aux(A_ML); ML_Operator_Destroy(&A_ML); return 0; }