bool probing_test(Epetra_CrsMatrix & in_mat, bool build_list){ const Epetra_CrsGraph & graph=in_mat.Graph(); Teuchos::ParameterList main, zoltan; if(build_list){ // zoltan.set("DISTANCE","2"); main.set("ZOLTAN",zoltan); } Isorropia::Epetra::Prober prober(Teuchos::rcp<const Epetra_CrsGraph>(&graph,false),main); Epetra_CrsMatrix out_mat(Copy,graph); int rv=prober.probe(in_mat,out_mat); if(rv!=0) {printf("ERROR: probing failed\n");return false;} #ifdef HAVE_EPETRAEXT EpetraExt::MatrixMatrix::Add(in_mat,false,1,out_mat,-1); double nrm=out_mat.NormInf()/in_mat.NormInf(); if(!in_mat.Comm().MyPID()) printf("diff norm = %22.16e\n",nrm); if(nrm < 1e-12) return true; else return false; #endif return true; }
// ================================================ ====== ==== ==== == = // Computes C= <me> * A int ML_Epetra::ML_RefMaxwell_11_Operator::MatrixMatrix_Multiply(const Epetra_CrsMatrix & A, Epetra_CrsMatrix **C) const { ML_Comm* comm; ML_Comm_Create(&comm); #ifdef ML_MPI const Epetra_MpiComm *epcomm = dynamic_cast<const Epetra_MpiComm*>(&(A.Comm())); // Get the MPI communicator, as it may not be MPI_COMM_W0RLD, and update the ML comm object if (epcomm) ML_Comm_Set_UsrComm(comm,epcomm->Comm()); #endif ML_Operator *C_; int rv=MatrixMatrix_Multiply(A,comm,&C_); Epetra_CrsMatrix_Wrap_ML_Operator(C_,*Comm_,*DomainMap_,C,Copy,A.IndexBase()); ML_Operator_Destroy(&C_); ML_Comm_Destroy(&comm); return rv; }/*end MatrixMatrix_Multiply*/
//============================================================================== Ifpack_CrsIct::Ifpack_CrsIct(const Epetra_CrsMatrix & A, double Droptol, int Lfil) : A_(A), Comm_(A.Comm()), Allocated_(false), ValuesInitialized_(false), Factored_(false), Condest_(-1.0), Athresh_(0.0), Rthresh_(1.0), Droptol_(Droptol), Lfil_(Lfil), LevelOverlap_(0), OverlapMode_(Zero), Aict_(0), Lict_(0), Ldiag_(0) { Allocate(); }
int TestMultiLevelPreconditioner(char ProblemType[], Teuchos::ParameterList & MLList, Epetra_LinearProblem & Problem, double & TotalErrorResidual, double & TotalErrorExactSol) { Epetra_MultiVector* lhs = Problem.GetLHS(); Epetra_MultiVector* rhs = Problem.GetRHS(); Epetra_CrsMatrix* A = dynamic_cast<Epetra_CrsMatrix*>(Problem.GetMatrix()); int PID = A->Comm().MyPID(); int numProcs = A->Comm().NumProc(); RCP<const Epetra_RowMatrix> Arcp = Teuchos::rcp(A, false); double n1, n2,nf; // ======================================== // // create a rhs corresponding to lhs or 1's // // ======================================== // lhs->PutScalar(1.0); A->Multiply(false,*lhs,*rhs); lhs->PutScalar(0.0); MLList.set("ML output", 0); RowMatrixToMatlabFile("mat_f.dat",*A); MultiVectorToMatrixMarketFile("lhs_f.dat",*lhs,0,0,false); MultiVectorToMatrixMarketFile("rhs_f.dat",*rhs,0,0,false); Epetra_Time Time(A->Comm()); /* Build the Zoltan list - Group #1 */ ParameterList Zlist1,Sublist1; Sublist1.set("DEBUG_LEVEL","0"); Sublist1.set("NUM_GLOBAL_PARTITIONS","2"); Zlist1.set("Zoltan",Sublist1); /* Start Isorropia's Ninja Magic - Group #1 */ RefCountPtr<Isorropia::Epetra::Partitioner> partitioner1 = Isorropia::Epetra::create_partitioner(Arcp, Zlist1); Isorropia::Epetra::Redistributor rd1(partitioner1); Teuchos::RCP<Epetra_CrsMatrix> ResA1=rd1.redistribute(*A); Teuchos::RCP<Epetra_MultiVector> ResX1=rd1.redistribute(*lhs); Teuchos::RCP<Epetra_MultiVector> ResB1=rd1.redistribute(*rhs); RestrictedCrsMatrixWrapper RW1; RW1.restrict_comm(ResA1); RestrictedMultiVectorWrapper RX1,RB1; RX1.restrict_comm(ResX1); RB1.restrict_comm(ResB1); /* Build the Zoltan list - Group #2 */ ParameterList Zlist2,Sublist2; Sublist2.set("DEBUG_LEVEL","0"); if(PID > 1) Sublist2.set("NUM_LOCAL_PARTITIONS","1"); else Sublist2.set("NUM_LOCAL_PARTITIONS","0"); Zlist2.set("Zoltan",Sublist2); /* Start Isorropia's Ninja Magic - Group #2 */ RefCountPtr<Isorropia::Epetra::Partitioner> partitioner2 = Isorropia::Epetra::create_partitioner(Arcp, Zlist2); Isorropia::Epetra::Redistributor rd2(partitioner2); Teuchos::RCP<Epetra_CrsMatrix> ResA2=rd2.redistribute(*A); Teuchos::RCP<Epetra_MultiVector> ResX2=rd2.redistribute(*lhs); Teuchos::RCP<Epetra_MultiVector> ResB2=rd2.redistribute(*rhs); RestrictedCrsMatrixWrapper RW2; RW2.restrict_comm(ResA2); RestrictedMultiVectorWrapper RX2,RB2; RX2.restrict_comm(ResX2); RB2.restrict_comm(ResB2); if(RW1.RestrictedProcIsActive()){ Teuchos::RCP<Epetra_CrsMatrix> SubA1 = RW1.RestrictedMatrix(); Teuchos::RCP<Epetra_MultiVector> SubX1 = RX1.RestrictedMultiVector(); Teuchos::RCP<Epetra_MultiVector> SubB1 = RB1.RestrictedMultiVector(); ML_Epetra::MultiLevelPreconditioner * SubPrec1 = new ML_Epetra::MultiLevelPreconditioner(*SubA1, MLList, true); Epetra_LinearProblem Problem1(&*SubA1,&*SubX1,&*SubB1); AztecOO solver1(Problem1); solver1.SetPrecOperator(SubPrec1); solver1.SetAztecOption(AZ_solver, AZ_gmres); solver1.SetAztecOption(AZ_output, 32); solver1.SetAztecOption(AZ_kspace, 160); solver1.Iterate(1550, 1e-12); delete SubPrec1; } else{ Teuchos::RCP<Epetra_CrsMatrix> SubA2 = RW2.RestrictedMatrix(); Teuchos::RCP<Epetra_MultiVector> SubX2 = RX2.RestrictedMultiVector(); Teuchos::RCP<Epetra_MultiVector> SubB2 = RB2.RestrictedMultiVector(); ML_Epetra::MultiLevelPreconditioner * SubPrec2 = new ML_Epetra::MultiLevelPreconditioner(*SubA2, MLList, true); Epetra_LinearProblem Problem2(&*SubA2,&*SubX2,&*SubB2); AztecOO solver2(Problem2); solver2.SetPrecOperator(SubPrec2); solver2.SetAztecOption(AZ_solver, AZ_gmres); solver2.SetAztecOption(AZ_output, 32); solver2.SetAztecOption(AZ_kspace, 160); solver2.Iterate(1550, 1e-12); delete SubPrec2; } /* Post-processing exports */ Epetra_MultiVector ans1(*lhs), ans2(*lhs); rd1.redistribute_reverse(*ResX1,ans1); rd2.redistribute_reverse(*ResX2,ans2); /* Run on Full Problem */ A->Comm().Barrier(); ML_Epetra::MultiLevelPreconditioner * FullPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true); AztecOO solverF(Problem); solverF.SetPrecOperator(FullPrec); solverF.SetAztecOption(AZ_solver, AZ_gmres); solverF.SetAztecOption(AZ_output, 32); solverF.SetAztecOption(AZ_kspace, 160); solverF.Iterate(1550, 1e-12); delete FullPrec; /* Solution Comparison */ ans1.Update(1.0,*lhs,-1.0); ans2.Update(1.0,*lhs,-1.0); ans1.Norm2(&n1); ans2.Norm2(&n2); if(!PID) { printf("Norm Diff 1 = %6.4e\n",n1); printf("Norm Diff 2 = %6.4e\n",n2); } TotalErrorExactSol += n1 + n2; }
bool CrsMatrixInfo( const Epetra_CrsMatrix & A, ostream & os ) { int MyPID = A.Comm().MyPID(); // take care that matrix is already trasformed bool IndicesAreGlobal = A.IndicesAreGlobal(); if( IndicesAreGlobal == true ) { if( MyPID == 0 ) { os << "WARNING : matrix must be transformed to local\n"; os << " before calling CrsMatrixInfo\n"; os << " Now returning...\n"; } return false; } int NumGlobalRows = A.NumGlobalRows(); int NumGlobalNonzeros = A.NumGlobalNonzeros(); int NumGlobalCols = A.NumGlobalCols(); double NormInf = A.NormInf(); double NormOne = A.NormOne(); int NumGlobalDiagonals = A.NumGlobalDiagonals(); int GlobalMaxNumEntries = A.GlobalMaxNumEntries(); int IndexBase = A.IndexBase(); bool StorageOptimized = A.StorageOptimized(); bool LowerTriangular = A.LowerTriangular(); bool UpperTriangular = A.UpperTriangular(); bool NoDiagonal = A.NoDiagonal(); // these variables identifies quantities I have to compute, // since not provided by Epetra_CrsMatrix double MyFrobeniusNorm( 0.0 ), FrobeniusNorm( 0.0 ); double MyMinElement( DBL_MAX ), MinElement( DBL_MAX ); double MyMaxElement( DBL_MIN ), MaxElement( DBL_MIN ); double MyMinAbsElement( DBL_MAX ), MinAbsElement( DBL_MAX ); double MyMaxAbsElement( 0.0 ), MaxAbsElement( 0.0 ); int NumMyRows = A.NumMyRows(); int * NzPerRow = new int[NumMyRows]; int Row; // iterator on rows int Col; // iterator on cols int MaxNumEntries = A.MaxNumEntries(); double * Values = new double[MaxNumEntries]; int * Indices = new int[MaxNumEntries]; double Element, AbsElement; // generic nonzero element and its abs value int NumEntries; double * Diagonal = new double [NumMyRows]; // SumOffDiagonal is the sum of absolute values for off-diagonals double * SumOffDiagonal = new double [NumMyRows]; for( Row=0 ; Row<NumMyRows ; ++Row ) { SumOffDiagonal[Row] = 0.0; } int * IsDiagonallyDominant = new int [NumMyRows]; int GlobalRow; // cycle over all matrix elements for( Row=0 ; Row<NumMyRows ; ++Row ) { GlobalRow = A.GRID(Row); NzPerRow[Row] = A.NumMyEntries(Row); A.ExtractMyRowCopy(Row,NzPerRow[Row],NumEntries,Values,Indices); for( Col=0 ; Col<NumEntries ; ++Col ) { Element = Values[Col]; AbsElement = abs(Element); if( Element<MyMinElement ) MyMinElement = Element; if( Element>MyMaxElement ) MyMaxElement = Element; if( AbsElement<MyMinAbsElement ) MyMinAbsElement = AbsElement; if( AbsElement>MyMaxAbsElement ) MyMaxAbsElement = AbsElement; if( Indices[Col] == Row ) Diagonal[Row] = Element; else SumOffDiagonal[Row] += abs(Element); MyFrobeniusNorm += pow(Element,2); } } // analise storage per row int MyMinNzPerRow( NumMyRows ), MinNzPerRow( NumMyRows ); int MyMaxNzPerRow( 0 ), MaxNzPerRow( 0 ); for( Row=0 ; Row<NumMyRows ; ++Row ) { if( NzPerRow[Row]<MyMinNzPerRow ) MyMinNzPerRow=NzPerRow[Row]; if( NzPerRow[Row]>MyMaxNzPerRow ) MyMaxNzPerRow=NzPerRow[Row]; } // a test to see if matrix is diagonally-dominant int MyDiagonalDominance( 0 ), DiagonalDominance( 0 ); int MyWeakDiagonalDominance( 0 ), WeakDiagonalDominance( 0 ); for( Row=0 ; Row<NumMyRows ; ++Row ) { if( abs(Diagonal[Row])>SumOffDiagonal[Row] ) ++MyDiagonalDominance; else if( abs(Diagonal[Row])==SumOffDiagonal[Row] ) ++MyWeakDiagonalDominance; } // reduction operations A.Comm().SumAll(&MyFrobeniusNorm, &FrobeniusNorm, 1); A.Comm().MinAll(&MyMinElement, &MinElement, 1); A.Comm().MaxAll(&MyMaxElement, &MaxElement, 1); A.Comm().MinAll(&MyMinAbsElement, &MinAbsElement, 1); A.Comm().MaxAll(&MyMaxAbsElement, &MaxAbsElement, 1); A.Comm().MinAll(&MyMinNzPerRow, &MinNzPerRow, 1); A.Comm().MaxAll(&MyMaxNzPerRow, &MaxNzPerRow, 1); A.Comm().SumAll(&MyDiagonalDominance, &DiagonalDominance, 1); A.Comm().SumAll(&MyWeakDiagonalDominance, &WeakDiagonalDominance, 1); // free memory delete Values; delete Indices; delete Diagonal; delete SumOffDiagonal; delete IsDiagonallyDominant; delete NzPerRow; // simply no output for MyPID>0, only proc 0 write on os if( MyPID != 0 ) return true; os << "*** general Information about the matrix\n"; os << "Number of Global Rows = " << NumGlobalRows << endl; os << "Number of Global Cols = " << NumGlobalCols << endl; os << "is the matrix square = " << ((NumGlobalRows==NumGlobalCols)?"yes":"no") << endl; os << "||A||_\\infty = " << NormInf << endl; os << "||A||_1 = " << NormOne << endl; os << "||A||_F = " << sqrt(FrobeniusNorm) << endl; os << "Number of nonzero diagonal entries = " << NumGlobalDiagonals << "( " << 1.0* NumGlobalDiagonals/NumGlobalRows*100 << " %)\n"; os << "Nonzero per row : min = " << MinNzPerRow << " average = " << 1.0*NumGlobalNonzeros/NumGlobalRows << " max = " << MaxNzPerRow << endl; os << "Maximum number of nonzero elements/row = " << GlobalMaxNumEntries << endl; os << "min( a_{i,j} ) = " << MinElement << endl; os << "max( a_{i,j} ) = " << MaxElement << endl; os << "min( abs(a_{i,j}) ) = " << MinAbsElement << endl; os << "max( abs(a_{i,j}) ) = " << MaxAbsElement << endl; os << "Number of diagonal dominant rows = " << DiagonalDominance << " (" << 100.0*DiagonalDominance/NumGlobalRows << " % of total)\n"; os << "Number of weakly diagonal dominant rows = " << WeakDiagonalDominance << " (" << 100.0*WeakDiagonalDominance/NumGlobalRows << " % of total)\n"; os << "*** Information about the Trilinos storage\n"; os << "Base Index = " << IndexBase << endl; os << "is storage optimized = " << ((StorageOptimized==true)?"yes":"no") << endl; os << "are indices global = " << ((IndicesAreGlobal==true)?"yes":"no") << endl; os << "is matrix lower triangular = " << ((LowerTriangular==true)?"yes":"no") << endl; os << "is matrix upper triangular = " << ((UpperTriangular==true)?"yes":"no") << endl; os << "are there diagonal entries = " << ((NoDiagonal==false)?"yes":"no") << endl; return true; }
/* Computes the approximate Schur complement for the wide separator using guided probing*/ Teuchos::RCP<Epetra_CrsMatrix> computeSchur_GuidedProbing ( shylu_config *config, shylu_symbolic *ssym, // symbolic structure shylu_data *data, // numeric structure Epetra_Map *localDRowMap ) { int i; double relative_thres = config->relative_threshold; Epetra_CrsMatrix *G = ssym->G.getRawPtr(); Epetra_CrsMatrix *R = ssym->R.getRawPtr(); Epetra_LinearProblem *LP = ssym->LP.getRawPtr(); Amesos_BaseSolver *solver = ssym->Solver.getRawPtr(); Ifpack_Preconditioner *ifSolver = ssym->ifSolver.getRawPtr(); Epetra_CrsMatrix *C = ssym->C.getRawPtr(); // Need to create local G (block diagonal portion) , R, C // Get row map of G Epetra_Map CrMap = C->RowMap(); int *c_rows = CrMap.MyGlobalElements(); int *c_cols = (C->ColMap()).MyGlobalElements(); //int c_totalElems = CrMap.NumGlobalElements(); int c_localElems = CrMap.NumMyElements(); int c_localcolElems = (C->ColMap()).NumMyElements(); Epetra_Map GrMap = G->RowMap(); int *g_rows = GrMap.MyGlobalElements(); //int g_totalElems = GrMap.NumGlobalElements(); int g_localElems = GrMap.NumMyElements(); Epetra_Map RrMap = R->RowMap(); int *r_rows = RrMap.MyGlobalElements(); int *r_cols = (R->ColMap()).MyGlobalElements(); //int r_totalElems = RrMap.NumGlobalElements(); int r_localElems = RrMap.NumMyElements(); int r_localcolElems = (R->ColMap()).NumMyElements(); Epetra_SerialComm LComm; Epetra_Map C_localRMap (-1, c_localElems, c_rows, 0, LComm); Epetra_Map C_localCMap (-1, c_localcolElems, c_cols, 0, LComm); Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm); Epetra_Map R_localRMap (-1, r_localElems, r_rows, 0, LComm); Epetra_Map R_localCMap (-1, r_localcolElems, r_cols, 0, LComm); //cout << "#local rows" << g_localElems << "#non zero local cols" << c_localcolElems << endl; #ifdef DEBUG cout << "DEBUG MODE" << endl; int nrows = C->RowMap().NumMyElements(); assert(nrows == localDRowMap->NumGlobalElements()); int gids[nrows], gids1[nrows]; C_localRMap.MyGlobalElements(gids); localDRowMap->MyGlobalElements(gids1); cout << "Comparing R's domain map with D's row map" << endl; for (int i = 0; i < nrows; i++) { assert(gids[i] == gids1[i]); } #endif int nentries1, gid; // maxentries is the maximum of all three possible matrices as the arrays // are reused between the three int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries()); maxentries = max(maxentries, G->MaxNumEntries()); double *values1 = new double[maxentries]; double *values2 = new double[maxentries]; double *values3 = new double[maxentries]; int *indices1 = new int[maxentries]; int *indices2 = new int[maxentries]; int *indices3 = new int[maxentries]; //cout << "Creating local matrices" << endl; int err; Epetra_CrsMatrix localC(Copy, C_localRMap, C->MaxNumEntries(), false); for (i = 0; i < c_localElems ; i++) { gid = c_rows[i]; err = C->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); assert (err == 0); //if (nentries1 > 0) // TODO: Later //{ err = localC.InsertGlobalValues(gid, nentries1, values1, indices1); assert (err == 0); //} } localC.FillComplete(G_localRMap, C_localRMap); //cout << "Created local C matrix" << endl; Epetra_CrsMatrix localR(Copy, R_localRMap, R->MaxNumEntries(), false); for (i = 0; i < r_localElems ; i++) { gid = r_rows[i]; R->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); localR.InsertGlobalValues(gid, nentries1, values1, indices1); } localR.FillComplete(*localDRowMap, R_localRMap); //cout << "Created local R matrix" << endl; // Sbar - Approximate Schur complement Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix( Copy, GrMap, g_localElems)); // Include only the block diagonal elements of G in localG Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false); int cnt, scnt; for (i = 0; i < g_localElems ; i++) { gid = g_rows[i]; G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); cnt = 0; scnt = 0; for (int j = 0 ; j < nentries1 ; j++) { if (G->LRID(indices1[j]) != -1) { values2[cnt] = values1[j]; indices2[cnt++] = indices1[j]; } else { // Add it to Sbar immediately values3[scnt] = values1[j]; indices3[scnt++] = indices1[j]; } } localG.InsertGlobalValues(gid, cnt, values2, indices2); Sbar->InsertGlobalValues(gid, scnt, values3, indices3); } localG.FillComplete(); cout << "Created local G matrix" << endl; int nvectors = 16; ShyLU_Probing_Operator probeop(config, ssym, &localG, &localR, LP, solver, ifSolver, &localC, localDRowMap, nvectors); #ifdef DUMP_MATRICES //ostringstream fnamestr; //fnamestr << "localC" << C->Comm().MyPID() << ".mat"; //string Cfname = fnamestr.str(); //EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC); //Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm()); //EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg = //new EpetraExt::CrsMatrix_Reindex( defMapg ); //Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG ); //ReIdx_MatTransg->fwd(); //EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G); #endif //cout << " totalElems in Schur Complement" << totalElems << endl; //cout << myPID << " localElems" << localElems << endl; // **************** Two collectives here ********************* #ifdef TIMING_OUTPUT Teuchos::Time ftime("setup time"); #endif #ifdef TIMING_OUTPUT Teuchos::Time app_time("setup time"); #endif Teuchos::RCP<Epetra_CrsGraph> lSGraph = Teuchos::RCP<Epetra_CrsGraph> ( new Epetra_CrsGraph(Copy, G_localRMap, maxentries)); if (data->num_compute % config->reset_iter == 0) { int nentries; // size > maxentries as there could be fill // TODO: Currently the size of the two arrays can be one, Even if we switch // the loop below the size of the array required is nvectors. Fix it double *values = new double[g_localElems]; int *indices = new int[g_localElems]; double *vecvalues; int dropped = 0; double *maxvalue = new double[nvectors]; #ifdef TIMING_OUTPUT ftime.start(); #endif int findex = g_localElems / nvectors ; int cindex; // int mypid = C->Comm().MyPID(); // unused Epetra_MultiVector probevec(G_localRMap, nvectors); Epetra_MultiVector Scol(G_localRMap, nvectors); for (i = 0 ; i < findex*nvectors ; i+=nvectors) { probevec.PutScalar(0.0); // TODO: Move it out for (int k = 0; k < nvectors; k++) { cindex = k+i; // TODO: Can do better than this, just need to go to the column map // of C, there might be null columns in C // Not much of use for Shasta 2x2 .. Later. probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0); //if (mypid == 0) //cout << "Changing row to 1.0 " << g_rows[cindex] << endl; } #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif Scol.MaxValue(maxvalue); for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops { cindex = k+i; vecvalues = Scol[k]; //cout << "MAX" << maxvalue << endl; for (int j = 0 ; j < g_localElems ; j++) { nentries = 0; // inserting one entry in each row for now if (g_rows[cindex] == g_rows[j]) // diagonal entry { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[cindex]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres) { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[cindex]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else { if (vecvalues[j] != 0.0) { dropped++; //cout << "vecvalues[j]" << vecvalues[j] << // " max" << maxvalue[k] << endl; } } } } } probeop.ResetTempVectors(1); for ( ; i < g_localElems ; i++) { // TODO: Can move the next two decalarations outside the loop Epetra_MultiVector probevec(G_localRMap, 1); Epetra_MultiVector Scol(G_localRMap, 1); probevec.PutScalar(0.0); // TODO: Can do better than this, just need to go to the column map // of C, there might be null columns in C probevec.ReplaceGlobalValue(g_rows[i], 0, 1.0); #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif vecvalues = Scol[0]; Scol.MaxValue(maxvalue); //cout << "MAX" << maxvalue << endl; for (int j = 0 ; j < g_localElems ; j++) { nentries = 0; // inserting one entry in each row for now if (g_rows[i] == g_rows[j]) // diagonal entry { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[i]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres) { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[i]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else { if (vecvalues[j] != 0.0) dropped++; } } } #ifdef TIMING_OUTPUT ftime.stop(); cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl; ftime.reset(); #endif #ifdef TIMING_OUTPUT cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl; #endif probeop.PrintTimingInfo(); Sbar->FillComplete(); lSGraph->FillComplete(); data->localSbargraph = lSGraph; #ifdef DUMP_MATRICES Epetra_Map defMap2(-1, g_localElems, 0, C->Comm()); EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 = new EpetraExt::CrsMatrix_Reindex( defMap2 ); Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar ); ReIdx_MatTrans2->fwd(); EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S); #endif cout << "#dropped entries" << dropped << endl; delete[] values; delete[] indices; delete[] maxvalue; } else { if (((data->num_compute-1) % config->reset_iter) == 0) { // We recomputed the Schur complement with dropping for the last // compute. Reset the prober with the new orthogonal vectors for // the Sbar from the previous iteration. Teuchos::ParameterList pList; Teuchos::RCP<Isorropia::Epetra::Prober> gprober = Teuchos::RCP<Isorropia::Epetra::Prober> (new Isorropia::Epetra::Prober( data->localSbargraph.getRawPtr(), pList, false)); gprober->color(); data->guided_prober = gprober; } // Use the prober to probe the probeop for the sparsity pattern // add that to Sbar and call Fill complete int nvectors = data->guided_prober->getNumOrthogonalVectors(); cout << "Number of Orthogonal Vectors for guided probing" << nvectors << endl; probeop.ResetTempVectors(nvectors); Teuchos::RCP<Epetra_CrsMatrix> blockdiag_Sbar = data->guided_prober->probe(probeop); int maxentries = blockdiag_Sbar->GlobalMaxNumEntries(); int *indices = new int[maxentries]; double *values = new double[maxentries]; int numentries; for (int i = 0; i < blockdiag_Sbar->NumGlobalRows() ; i++) { int gid = blockdiag_Sbar->GRID(i); blockdiag_Sbar->ExtractGlobalRowCopy(gid, maxentries, numentries, values, indices); Sbar->InsertGlobalValues(gid, numentries, values, indices); } Sbar->FillComplete(); delete[] indices; delete[] values; } delete[] values1; delete[] indices1; delete[] values2; delete[] indices2; delete[] values3; delete[] indices3; return Sbar; }
int NOX::Epetra::TestCompare::testCrsMatrices( const Epetra_CrsMatrix& mat , const Epetra_CrsMatrix& mat_expected , double rtol, double atol , const std::string& name , bool enforceStructure ) { if (utils.isPrintType(NOX::Utils::TestDetails)) os << std::endl << "\tChecking " << name << ": "; int passed = 0; if( !mat_expected.RowMap().SameAs( mat.RowMap() ) ) { passed = 1; os << "Failed." << std::endl; os << std::endl << "\t\tRow maps are not compatible." << std::endl; return passed; } int numEntries1, numEntries2 ; int * columns1 , * columns2 ; double * values1 , * values2 ; int chkSize = 0 ; double maxVal = 0.0 ; double infNorm = 0.0 ; for( int row = 0; row < mat_expected.NumMyRows(); ++row ) { mat_expected.ExtractMyRowView(row, numEntries1, values1, columns1); mat.ExtractMyRowView (row, numEntries2, values2, columns2); if( numEntries1 != numEntries2 ) { if( enforceStructure ) { os << std::endl << "\t\t\t"; } else os << std::endl << "\t\tWARNING: "; os << "Matrix size is incompatible for Local Row " << row << "\n\t\t\t..... expected " << numEntries1 << " columns, found " << numEntries2 << std::endl; chkSize = 1; } mat.Comm().SumAll( &chkSize, &passed, 1 ); if( 0 != passed ) { if( enforceStructure ) { os << "Failed." << std::endl; return passed; } else { chkSize = 0; passed = 0; } } // Comapre column indices and values int baseCol = 0 ; int testCol = 0 ; int chkCol = 0 ; double baseVal = 0.0 ; double testVal = 0.0 ; double chkVal = 0.0 ; for( int col = 0; col < numEntries1; ++col ) { baseCol = columns1[col]; testCol = columns2[col]; baseVal = values1 [col]; testVal = values2 [col]; if( baseCol != testCol ) { if( enforceStructure ) { os << std::endl << "\t\t\t"; } else os << std::endl << "\t\tWARNING: "; os << "Column index for Local Row " << row << " is incompatible." << "\n\t\t\t..... expected " << baseCol << " , found " << testCol << std::endl; chkCol = 1; } mat.Comm().SumAll( &chkCol, &passed, 1 ); if( 0 != passed ) { if( enforceStructure ) { os << "Failed." << std::endl; return passed; } else { chkCol = 0; passed = 0; continue; // skip pvalue check } } chkVal = fabs( testVal - baseVal ) / (atol + rtol * fabs(baseVal)); mat.Comm().MaxAll( &chkVal, &maxVal, 1 ); if( maxVal > infNorm ) infNorm = maxVal; if( 1 < maxVal ) break; } if( 1 < maxVal ) break; } if( 1 < maxVal ) passed = 1; // false by convention in NOX::TestCompare else passed = 0; // true by convention in NOX::TestCompare if (utils.isPrintType(NOX::Utils::TestDetails)) { if( 0 == passed) os << "Passed." << std::endl; else os << "Failed." << std::endl; os << "\t\tComputed norm: " << utils.sciformat(infNorm) << std::endl << "\t\tRelative Tolerance: " << utils.sciformat(rtol) << std::endl << "\t\tAbsolute Tolerance: " << utils.sciformat(rtol) << std::endl; } return passed; }
// // Diagonal: 0=no change, 1=eliminate entry // from the map for the largest row element in process 0 // 2=add diagonal entries to the matrix, with a zero value // (assume row map contains all diagonal entries). // // ReindexRowMap: // 0=no change, 1= add 2 (still contiguous), 2=non-contiguous // // ReindexColMap // 0=same as RowMap, 1=add 4 - Different From RowMap, but contiguous) // // RangeMap: // 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map // // DomainMap: // 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map // RCP<Epetra_CrsMatrix> NewMatNewMap(Epetra_CrsMatrix& In, int Diagonal, int ReindexRowMap, int ReindexColMap, int RangeMapType, int DomainMapType ) { // // If we are making no change, return the original matrix (which has a linear map) // #if 0 std::cout << __FILE__ << "::" << __LINE__ << " " << Diagonal << " " << ReindexRowMap << " " << ReindexColMap << " " << RangeMapType << " " << DomainMapType << " " << std::endl ; #endif if ( Diagonal + ReindexRowMap + ReindexColMap + RangeMapType + DomainMapType == 0 ) { RCP<Epetra_CrsMatrix> ReturnOrig = rcp( &In, false ); return ReturnOrig ; } // // Diagonal==2 is used for a different purpose - // Making sure that the diagonal of the matrix is non-empty. // Note: The diagonal must exist in In.RowMap(). // if ( Diagonal == 2 ) { assert( ReindexRowMap==0 && ReindexColMap == 0 ) ; } int (*RowPermute)(int in) = 0; int (*ColPermute)(int in) = 0; assert( Diagonal >= 0 && Diagonal <= 2 ); assert( ReindexRowMap>=0 && ReindexRowMap<=2 ); assert( ReindexColMap>=0 && ReindexColMap<=1 ); assert( RangeMapType>=0 && RangeMapType<=3 ); assert( DomainMapType>=0 && DomainMapType<=3 ); Epetra_Map DomainMap = In.DomainMap(); Epetra_Map RangeMap = In.RangeMap(); Epetra_Map ColMap = In.ColMap(); Epetra_Map RowMap = In.RowMap(); int NumMyRowElements = RowMap.NumMyElements(); int NumMyColElements = ColMap.NumMyElements(); int NumMyRangeElements = RangeMap.NumMyElements(); int NumMyDomainElements = DomainMap.NumMyElements(); int NumGlobalRowElements = RowMap.NumGlobalElements(); int NumGlobalColElements = ColMap.NumGlobalElements(); int NumGlobalRangeElements = RangeMap.NumGlobalElements(); int NumGlobalDomainElements = DomainMap.NumGlobalElements(); assert( NumGlobalRangeElements == NumGlobalDomainElements ) ; std::vector<int> MyGlobalRowElements( NumMyRowElements ) ; std::vector<int> NumEntriesPerRow( NumMyRowElements ) ; std::vector<int> MyPermutedGlobalRowElements( NumMyRowElements ) ; std::vector<int> MyGlobalColElements( NumMyColElements ) ; std::vector<int> MyPermutedGlobalColElements( NumMyColElements ) ; // Used to create the column map std::vector<int> MyPermutedGlobalColElementTable( NumMyColElements ) ; // To convert local indices to global std::vector<int> MyGlobalRangeElements( NumMyRangeElements ) ; std::vector<int> MyPermutedGlobalRangeElements( NumMyRangeElements ) ; std::vector<int> MyGlobalDomainElements( NumMyDomainElements ) ; std::vector<int> MyPermutedGlobalDomainElements( NumMyDomainElements ) ; RowMap.MyGlobalElements(&MyGlobalRowElements[0]); ColMap.MyGlobalElements(&MyGlobalColElements[0]); RangeMap.MyGlobalElements(&MyGlobalRangeElements[0]); DomainMap.MyGlobalElements(&MyGlobalDomainElements[0]); switch( ReindexRowMap ) { case 0: RowPermute = &NoPermute ; break; case 1: RowPermute = &SmallRowPermute ; break; case 2: RowPermute = BigRowPermute ; break; } switch( ReindexColMap ) { case 0: ColPermute = RowPermute ; break; case 1: ColPermute = &SmallColPermute ; break; } // // Create Serial Range and Domain Maps based on the permuted indexing // int nlocal = 0; if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements; std::vector<int> AllIDs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDs[i] = (*RowPermute)( i ) ; Epetra_Map SerialRangeMap( -1, nlocal, &AllIDs[0], 0, In.Comm()); std::vector<int> AllIDBs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDBs[i] = (*ColPermute)( i ) ; Epetra_Map SerialDomainMap( -1, nlocal, &AllIDBs[0], 0, In.Comm()); // // Create Bizarre Range and Domain Maps based on the permuted indexing // These are nearly serial, having all but one element on process 0 // The goal here is to make sure that we can use Domain and Range maps // that are neither serial, nor distributed in the normal manner. // std::vector<int> AllIDCs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDCs[i] = (*ColPermute)( i ) ; if ( In.Comm().NumProc() > 1 ) { if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1; if (In.Comm().MyPID()==1) { nlocal = 1; AllIDCs[0] = (*ColPermute)( NumGlobalRangeElements - 1 ); } } int iam = In.Comm().MyPID(); Epetra_Map BizarreDomainMap( -1, nlocal, &AllIDCs[0], 0, In.Comm()); std::vector<int> AllIDDs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDDs[i] = (*RowPermute)( i ) ; if ( In.Comm().NumProc() > 1 ) { if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1; if (In.Comm().MyPID()==1) { nlocal = 1; AllIDDs[0] = (*RowPermute)( NumGlobalRangeElements -1 ) ; } } Epetra_Map BizarreRangeMap( -1, nlocal, &AllIDDs[0], 0, In.Comm()); // // Compute the column map // // If Diagonal==1, remove the column corresponding to the last row owned // by process 0. Removing this column from a tridiagonal matrix, leaves // a disconnected, but non-singular matrix. // int NumMyColElementsOut = 0 ; int NumGlobalColElementsOut ; if ( Diagonal == 1 ) NumGlobalColElementsOut = NumGlobalColElements-1; else NumGlobalColElementsOut = NumGlobalColElements; if ( Diagonal == 1 && iam==0 ) { for ( int i=0; i < NumMyColElements ; i++ ) { if ( MyGlobalColElements[i] != MyGlobalRowElements[NumMyRowElements-1] ) { MyPermutedGlobalColElements[NumMyColElementsOut++] = (*ColPermute)( MyGlobalColElements[i] ) ; } } assert( NumMyColElementsOut == NumMyColElements-1 ); } else { for ( int i=0; i < NumMyColElements ; i++ ) MyPermutedGlobalColElements[i] = (*ColPermute)( MyGlobalColElements[i] ) ; NumMyColElementsOut = NumMyColElements ; if ( Diagonal == 2 ) { // For each row, make sure that the column map has this row in it, // if it doesn't, add it to the column map. // Note: MyPermutedGlobalColElements == MyGlobalColElements when // Diagonal==2 because ( Diagonal == 2 ) implies: // ReindexRowMap==0 && ReindexColMap == 0 - see assert above for ( int i=0; i < NumMyRowElements ; i++ ) { bool MissingDiagonal = true; for ( int j=0; j < NumMyColElements; j++ ) { if ( MyGlobalRowElements[i] == MyGlobalColElements[j] ) { MissingDiagonal = false; } } if ( MissingDiagonal ) { MyPermutedGlobalColElements.resize(NumMyColElements+1); MyPermutedGlobalColElements[NumMyColElementsOut] = MyGlobalRowElements[i]; NumMyColElementsOut++; } } In.Comm().SumAll(&NumMyColElementsOut,&NumGlobalColElementsOut,1); } } // // These tables are used both as the permutation tables and to create the maps. // for ( int i=0; i < NumMyColElements ; i++ ) MyPermutedGlobalColElementTable[i] = (*ColPermute)( MyGlobalColElements[i] ) ; for ( int i=0; i < NumMyRowElements ; i++ ) MyPermutedGlobalRowElements[i] = (*RowPermute)( MyGlobalRowElements[i] ) ; for ( int i=0; i < NumMyRangeElements ; i++ ) MyPermutedGlobalRangeElements[i] = (*RowPermute)( MyGlobalRangeElements[i] ) ; for ( int i=0; i < NumMyDomainElements ; i++ ) MyPermutedGlobalDomainElements[i] = (*ColPermute)( MyGlobalDomainElements[i] ) ; RCP<Epetra_Map> PermutedRowMap = rcp( new Epetra_Map( NumGlobalRowElements, NumMyRowElements, &MyPermutedGlobalRowElements[0], 0, In.Comm() ) ); RCP<Epetra_Map> PermutedColMap = rcp( new Epetra_Map( NumGlobalColElementsOut, NumMyColElementsOut, &MyPermutedGlobalColElements[0], 0, In.Comm() ) ); RCP<Epetra_Map> PermutedRangeMap = rcp( new Epetra_Map( NumGlobalRangeElements, NumMyRangeElements, &MyPermutedGlobalRangeElements[0], 0, In.Comm() ) ); RCP<Epetra_Map> PermutedDomainMap = rcp( new Epetra_Map( NumGlobalDomainElements, NumMyDomainElements, &MyPermutedGlobalDomainElements[0], 0, In.Comm() ) ); // // These vectors are filled and then passed to InsertGlobalValues // std::vector<int> ThisRowIndices( In.MaxNumEntries() ); std::vector<double> ThisRowValues( In.MaxNumEntries() ); std::vector<int> PermutedGlobalColIndices( In.MaxNumEntries() ); //std::cout << __FILE__ << "::" <<__LINE__ << std::endl ; RCP<Epetra_CrsMatrix> Out = rcp( new Epetra_CrsMatrix( Copy, *PermutedRowMap, *PermutedColMap, 0 ) ); for (int i=0; i<NumMyRowElements; i++) { int NumIndicesThisRow = 0; assert( In.ExtractMyRowCopy( i, In.MaxNumEntries(), NumIndicesThisRow, &ThisRowValues[0], &ThisRowIndices[0] ) == 0 ) ; for (int j = 0 ; j < NumIndicesThisRow ; j++ ) { PermutedGlobalColIndices[j] = MyPermutedGlobalColElementTable[ ThisRowIndices[j] ] ; } bool MissingDiagonal = false; if ( Diagonal==2 ) { // assert( MyGlobalRowElements[i] == MyPermutedGlobalRowElements[i] ); MissingDiagonal = true; for( int j =0 ; j < NumIndicesThisRow ; j++ ) { if ( PermutedGlobalColIndices[j] == MyPermutedGlobalRowElements[i] ) { MissingDiagonal = false ; } } #if 0 std::cout << __FILE__ << "::" << __LINE__ << " i = " << i << " MyPermutedGlobalRowElements[i] = " << MyPermutedGlobalRowElements[i] << " MissingDiagonal = " << MissingDiagonal << std::endl ; #endif } if ( MissingDiagonal ) { ThisRowValues.resize(NumIndicesThisRow+1) ; ThisRowValues[NumIndicesThisRow] = 0.0; PermutedGlobalColIndices.resize(NumIndicesThisRow+1); PermutedGlobalColIndices[NumIndicesThisRow] = MyPermutedGlobalRowElements[i] ; #if 0 std::cout << __FILE__ << "::" << __LINE__ << " i = " << i << "NumIndicesThisRow = " << NumIndicesThisRow << "ThisRowValues[NumIndicesThisRow = " << ThisRowValues[NumIndicesThisRow] << " PermutedGlobalColIndices[NumIndcesThisRow] = " << PermutedGlobalColIndices[NumIndicesThisRow] << std::endl ; #endif NumIndicesThisRow++ ; } assert( Out->InsertGlobalValues( MyPermutedGlobalRowElements[i], NumIndicesThisRow, &ThisRowValues[0], &PermutedGlobalColIndices[0] ) >= 0 ); } // Epetra_LocalMap ReplicatedMap( NumGlobalRangeElements, 0, In.Comm() ); RCP<Epetra_Map> OutRangeMap ; RCP<Epetra_Map> OutDomainMap ; switch( RangeMapType ) { case 0: OutRangeMap = PermutedRangeMap ; break; case 1: OutRangeMap = rcp(&SerialRangeMap, false); break; case 2: OutRangeMap = rcp(&BizarreRangeMap, false); break; case 3: OutRangeMap = rcp(&ReplicatedMap, false); break; } // switch( DomainMapType ) { switch( DomainMapType ) { case 0: OutDomainMap = PermutedDomainMap ; break; case 1: OutDomainMap = rcp(&SerialDomainMap, false); break; case 2: OutDomainMap = rcp(&BizarreDomainMap, false); break; case 3: OutDomainMap = rcp(&ReplicatedMap, false); break; } #if 0 assert(Out->FillComplete( *PermutedDomainMap, *PermutedRangeMap )==0); #else assert(Out->FillComplete( *OutDomainMap, *OutRangeMap )==0); #endif #if 0 std::cout << __FILE__ << "::" << __LINE__ << std::endl ; Out->Print( std::cout ) ; #endif return Out; }
int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix, int NumSameIDs, int NumRemoteIDs, const int * RemoteLIDs, int NumPermuteIDs, const int *PermuteToLIDs, const int *PermuteFromLIDs, int LenImports, char* Imports, int TargetNumRows, int TargetNumNonzeros, int * CSR_rowptr, int_type * CSR_colind, double * CSR_vals, const std::vector<int> &SourcePids, std::vector<int> &TargetPids) { // What we really need to know is where in the CSR arrays each row should start (aka the rowptr). // We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and // (c) Having each row copied into the right colind / values locations. // From Epetra_CrsMatrix UnpackAndCombine(): // Each segment of Exports will be filled by a packed row of information for each row as follows: // 1st int: GRID of row where GRID is the global row ID for the source matrix // next int: NumEntries, Number of indices in row. // next NumEntries: The actual indices for the row. int i,j,rv; int N = TargetNumRows; int mynnz = TargetNumNonzeros; int MyPID = SourceMatrix.Comm().MyPID(); int SizeofIntType = sizeof(int_type); // Zero the rowptr for(i=0; i<N+1; i++) CSR_rowptr[i]=0; // SameIDs: Always first, always in the same place for(i=0; i<NumSameIDs; i++) CSR_rowptr[i]=SourceMatrix.NumMyEntries(i); // PermuteIDs: Still local, but reordered for(i=0; i<NumPermuteIDs; i++) CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]); // RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); for(i=0; i<NumRemoteIDs; i++) { CSR_rowptr[RemoteLIDs[i]] += NumEntries; if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); } } } // If multiple procs contribute to a row; std::vector<int> NewStartRow(N+1); // Turn row length into a real CSR_rowptr int last_len = CSR_rowptr[0]; CSR_rowptr[0] = 0; for(i=1; i<N+1; i++){ int new_len = CSR_rowptr[i]; CSR_rowptr[i] = last_len + CSR_rowptr[i-1]; NewStartRow[i] = CSR_rowptr[i]; last_len = new_len; } // Preseed TargetPids with -1 for local if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz); TargetPids.assign(mynnz,-1); // Grab pointers for SourceMatrix int * Source_rowptr, * Source_colind; double * Source_vals; rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals); if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers"); // SameIDs: Copy the data over for(i=0; i<NumSameIDs; i++) { int FromRow = Source_rowptr[i]; int ToRow = CSR_rowptr[i]; NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i]; for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) { CSR_vals[ToRow + j - FromRow] = Source_vals[j]; CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]); TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1; } } // PermuteIDs: Copy the data over for(i=0; i<NumPermuteIDs; i++) { int FromLID = PermuteFromLIDs[i]; int FromRow = Source_rowptr[FromLID]; int ToRow = CSR_rowptr[PermuteToLIDs[i]]; NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID]; for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) { CSR_vals[ToRow + j - FromRow] = Source_vals[j]; CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]); TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1; } } // RemoteIDs: Loop structure following UnpackAndCombine if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); double* valptr = dintptr + IntSize; for (i=0; i<NumRemoteIDs; i++) { int ToLID = RemoteLIDs[i]; int StartRow = NewStartRow[ToLID]; NewStartRow[ToLID]+=NumEntries; double * values = valptr; int_type * Indices = intptr + 2; for(j=0; j<NumEntries; j++){ CSR_vals[StartRow + j] = values[j]; CSR_colind[StartRow + j] = Indices[2*j]; TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1; } if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); valptr = dintptr + IntSize; } } } return 0; }
int main(int argc, char *argv[]) { #ifdef EPETRA_MPI MPI_Init(&argc,&argv); Epetra_MpiComm comm(MPI_COMM_WORLD); #else Epetra_SerialComm comm; #endif int numprocs = comm.NumProc(); int localproc = comm.MyPID(); /////////////////////////////////////////////////// //First figure out whether verbose output is on. //(and if so, only turn it on for proc 0) bool verbose = false; if (localproc == 0) { verbose = argument_is_present("-v", argc, argv); } /////////////////////////////////////////////////// int local_n = 30; int global_n = numprocs*local_n; Epetra_Map emap(global_n, 0, comm); Epetra_CrsMatrix* A = create_and_fill_crs_matrix(emap); Epetra_Vector x(emap), b(emap); x.PutScalar(1.0); A->Multiply(false, x, b); x.PutScalar(0.0); double initial_norm = resid2norm(*A, x, b); if (verbose) { cout << "Initial 2-norm of b-A*x: "<<initial_norm<<endl; } int err = test_azoo_as_precond_op(*A, x, b, verbose); if (err != 0) { cout << "test_azoo_as_precond_op err, test FAILED."<<endl; return(err); } err = test_azoo_conv_anorm(*A, x, b, verbose); if (err != 0) { cout << "test_azoo_conv_anorm err, test FAILED."<<endl; return(err); } if (verbose) { cout << "testing AztecOO with AZ_conv = AZ_Anorm, and AZ_scaling = AZ_sym_diag" << endl; } err = test_azoo_conv_with_scaling(AZ_Anorm, AZ_sym_diag, A->Comm(), verbose); if (err != 0) { cout << "test_azoo_conv_with_scaling err, test FAILED."<<endl; return(err); } if (verbose) { cout << "testing AztecOO with AZ_conv = AZ_rhs, and AZ_scaling = AZ_sym_diag" << endl; } err = test_azoo_conv_with_scaling(AZ_rhs, AZ_sym_diag, A->Comm(), verbose); if (err != 0) { cout << "test_azoo_conv_with_scaling err, test FAILED."<<endl; return(err); } err = test_azoo_with_ilut(*A, x, b, verbose); if (err != 0) { cout << "test_azoo_with_ilut err, test FAILED."<<endl; return(err); } err = test_azoo_scaling(*A, x, b, verbose); if (err != 0) { cout << "test_azoo_scaling err="<<err<<", test FAILED."<<endl; return(err); } delete A; int* options = new int[AZ_OPTIONS_SIZE]; options[AZ_solver] = AZ_cg; options[AZ_subdomain_solve] = AZ_none; options[AZ_precond] = AZ_Jacobi; if (verbose) std::cout << "about to call test_AZ_iterate_AZ_pre_calc_AZ_reuse" <<std::endl; err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose); if (err != 0) { cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl; return(err); } options[AZ_solver] = AZ_cgs; options[AZ_subdomain_solve] = AZ_icc; options[AZ_precond] = AZ_dom_decomp; err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose); if (err != 0) { cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl; return(err); } options[AZ_solver] = AZ_gmres; options[AZ_subdomain_solve] = AZ_ilut; err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose); if (err != 0) { cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl; return(err); } options[AZ_solver] = AZ_tfqmr; options[AZ_subdomain_solve] = AZ_ilu; err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose); if (err != 0) { cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl; return(err); } options[AZ_solver] = AZ_bicgstab; options[AZ_subdomain_solve] = AZ_rilu; err = test_AZ_iterate_AZ_pre_calc_AZ_reuse(comm, options, verbose); if (err != 0) { cout << "test_AZ_iterate_AZ_pre_calc_AZ_reuse err, test FAILED."<<endl; return(err); } err = test_AZ_iterate_then_AZ_scale_f(comm, verbose); if (err != 0) { cout << "test_AZ_iterate_then_AZ_scale_f err, test FAILED."<<endl; return(err); } delete [] options; err = test_bug2554(comm, verbose); if (err != 0) { cout << "test_bug2554 err, test FAILED."<<endl; return(err); } err = test_bug2890(comm, verbose); if (err != 0) { cout << "test_bug2890 err, test FAILED."<<endl; return(err); } cout << "********* Test passed **********" << endl; #ifdef EPETRA_MPI MPI_Finalize() ; #endif return(0); }
// ================================================ ====== ==== ==== == = int ML_Epetra::RefMaxwell_Aggregate_Nodes(const Epetra_CrsMatrix & A, Teuchos::ParameterList & List, ML_Comm * ml_comm, std::string PrintMsg, ML_Aggregate_Struct *& MLAggr,ML_Operator *&P, int &NumAggregates){ /* Output level */ bool verbose, very_verbose; int OutputLevel = List.get("ML output", -47); if(OutputLevel == -47) OutputLevel = List.get("output", 1); if(OutputLevel>=15) very_verbose=verbose=true; if(OutputLevel > 5) {very_verbose=false;verbose=true;} else very_verbose=verbose=false; /* Wrap A in a ML_Operator */ ML_Operator* A_ML = ML_Operator_Create(ml_comm); ML_Operator_WrapEpetraCrsMatrix(const_cast<Epetra_CrsMatrix*>(&A),A_ML); /* Pull Teuchos Options */ std::string CoarsenType = List.get("aggregation: type", "Uncoupled"); double Threshold = List.get("aggregation: threshold", 0.0); int NodesPerAggr = List.get("aggregation: nodes per aggregate", ML_Aggregate_Get_OptimalNumberOfNodesPerAggregate()); bool UseAux = List.get("aggregation: aux: enable",false); double AuxThreshold = List.get("aggregation: aux: threshold",0.0); int MaxAuxLevels = List.get("aggregation: aux: max levels",10); ML_Aggregate_Create(&MLAggr); ML_Aggregate_Set_MaxLevels(MLAggr, 2); ML_Aggregate_Set_StartLevel(MLAggr, 0); ML_Aggregate_Set_Threshold(MLAggr, Threshold); ML_Aggregate_Set_MaxCoarseSize(MLAggr,1); MLAggr->cur_level = 0; ML_Aggregate_Set_Reuse(MLAggr); MLAggr->keep_agg_information = 1; P = ML_Operator_Create(ml_comm); /* Process Teuchos Options */ if (CoarsenType == "Uncoupled") ML_Aggregate_Set_CoarsenScheme_Uncoupled(MLAggr); else if (CoarsenType == "Uncoupled-MIS"){ ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr); } else if (CoarsenType == "METIS"){ ML_Aggregate_Set_CoarsenScheme_METIS(MLAggr); ML_Aggregate_Set_NodesPerAggr(0, MLAggr, 0, NodesPerAggr); }/*end if*/ else { if(!A.Comm().MyPID()) printf("%s Unsupported (1,1) block aggregation type(%s), resetting to uncoupled-mis\n",PrintMsg.c_str(),CoarsenType.c_str()); ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr); } /* Setup Aux Data */ if(UseAux) { A_ML->aux_data->enable=1; A_ML->aux_data->threshold=AuxThreshold; A_ML->aux_data->max_level=MaxAuxLevels; ML_Init_Aux(A_ML,List); if(verbose && !A.Comm().MyPID()) { printf("%s Using auxiliary matrix\n",PrintMsg.c_str()); printf("%s aux threshold = %e\n",PrintMsg.c_str(),A_ML->aux_data->threshold); } } /* Aggregate Nodes */ int printlevel=ML_Get_PrintLevel(); if(verbose) ML_Set_PrintLevel(10); NumAggregates = ML_Aggregate_Coarsen(MLAggr,A_ML, &P, ml_comm); if(verbose) ML_Set_PrintLevel(printlevel); if (NumAggregates == 0){ std::cerr << "Found 0 aggregates, perhaps the problem is too small." << std::endl; ML_CHK_ERR(-2); }/*end if*/ else if(very_verbose) printf("[%d] %s %d aggregates created invec_leng=%d\n",A.Comm().MyPID(),PrintMsg.c_str(),NumAggregates,P->invec_leng); if(verbose){ int globalAggs=0; A.Comm().SumAll(&NumAggregates,&globalAggs,1); if(!A.Comm().MyPID()) { printf("%s Aggregation threshold = %e\n",PrintMsg.c_str(),Threshold); printf("%s Global aggregates = %d\n",PrintMsg.c_str(),globalAggs); } } /* Cleanup */ ML_qr_fix_Destroy(); if(UseAux) ML_Finalize_Aux(A_ML); ML_Operator_Destroy(&A_ML); return 0; }