Teuchos::RCP<Epetra_Map> Albany::SolutionResponseFunction:: buildCulledMap(const Epetra_Map& x_map, const Teuchos::Array<int>& keepDOF) const { int numKeepDOF = std::accumulate(keepDOF.begin(), keepDOF.end(), 0); int Neqns = keepDOF.size(); int N = x_map.NumMyElements(); // x_map is map for solution vector TEUCHOS_ASSERT( !(N % Neqns) ); // Assume that all the equations for // a given node are on the assigned // processor. I.e. need to ensure // that N is exactly Neqns-divisible int nnodes = N / Neqns; // number of fem nodes int N_new = nnodes * numKeepDOF; // length of local x_new int *gids = x_map.MyGlobalElements(); // Fill local x_map into gids array Teuchos::Array<int> gids_new(N_new); int idx = 0; for ( int inode = 0; inode < N/Neqns ; ++inode) // For every node for ( int ieqn = 0; ieqn < Neqns; ++ieqn ) // Check every dof on the node if ( keepDOF[ieqn] == 1 ) // then want to keep this dof gids_new[idx++] = gids[(inode*Neqns)+ieqn]; // end cull Teuchos::RCP<Epetra_Map> x_new_map = Teuchos::rcp( new Epetra_Map( -1, N_new, &gids_new[0], 0, x_map.Comm() ) ); return x_new_map; }
// ============================================================================= Teuchos::RCP<Epetra_Map> VIO::EpetraMesh::Reader:: createComplexValuesMap_ ( const Epetra_Map & nodesMap ) const { // get view for the global indices of the global elements int numMyElements = nodesMap.NumMyElements(); Teuchos::ArrayRCP<int> myGlobalElements( numMyElements ); nodesMap.MyGlobalElements( myGlobalElements.getRawPtr() ); // Construct the map in such a way that all complex entries on processor K // are split up into real and imaginary part, which will both reside on // processor K again. int numMyComplexElements = 2*numMyElements; Teuchos::ArrayRCP<int> myComplexGlobalElements ( numMyComplexElements ); for ( int k = 0; k < numMyElements; k++ ) { myComplexGlobalElements[2*k ] = 2 * myGlobalElements[k]; myComplexGlobalElements[2*k+1] = 2 * myGlobalElements[k] + 1; } return Teuchos::rcp ( new Epetra_Map ( -1, myComplexGlobalElements.size(), myComplexGlobalElements.getRawPtr(), nodesMap.IndexBase(), nodesMap.Comm() ) ); }
TridiagonalCrsMatrix( const Epetra_Map & Map, double a, double diag, double c ) : Epetra_CrsMatrix(Copy,Map,3) { // global number of rows int NumGlobalElements = Map.NumGlobalElements(); // local number of rows int NumMyElements = Map.NumMyElements(); // get update list int * MyGlobalElements = new int [NumMyElements]; Map.MyGlobalElements( MyGlobalElements ); // Add rows one-at-a-time // Need some vectors to help // Off diagonal Values will always be -1 double *Values = new double[2]; Values[0] = a; Values[1] = c; int *Indices = new int[2]; int NumEntries; for( int i=0 ; i<NumMyElements; ++i ) { if (MyGlobalElements[i]==0) { Indices[0] = 1; NumEntries = 1; } else if (MyGlobalElements[i] == NumGlobalElements-1) { Indices[0] = NumGlobalElements-2; NumEntries = 1; } else { Indices[0] = MyGlobalElements[i]-1; Indices[1] = MyGlobalElements[i]+1; NumEntries = 2; } InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices); // Put in the diagonal entry InsertGlobalValues(MyGlobalElements[i], 1, &diag, MyGlobalElements+i); } // Finish up FillComplete(); delete [] MyGlobalElements; delete [] Values; delete [] Indices; }
//============================================================================== int Komplex_LinearProblem::MakeKomplexMap(const Epetra_Map & Map, Teuchos::RCP<Epetra_Map> & KMap) { if (Map.LinearMap()) { KMap = Teuchos::rcp(new Epetra_Map(-1, Map.NumMyElements()*2, Map.IndexBase(), Map.Comm())); } else { Epetra_IntSerialDenseVector KMapGIDs(Map.NumMyElements()*2); int * myGlobalElements = Map.MyGlobalElements(); for (int i=0; i<Map.NumMyElements(); i++) { KMapGIDs[2*i] = 2*myGlobalElements[i]; KMapGIDs[2*i+1] = 2*myGlobalElements[i]+1; } KMap = Teuchos::rcp(new Epetra_Map(-1, KMapGIDs.Length(), KMapGIDs.Values(), Map.IndexBase(), Map.Comm())); } return(0); }
//EpetraMap_To_TpetraMap: takes in Epetra_Map object, converts it to its equivalent Tpetra::Map object, //and returns an RCP pointer to this Tpetra::Map Teuchos::RCP<const Tpetra_Map> Petra::EpetraMap_To_TpetraMap(const Epetra_Map& epetraMap_, const Teuchos::RCP<const Teuchos::Comm<int> >& commT_) { const std::size_t numElements = Teuchos::as<std::size_t>(epetraMap_.NumMyElements()); const auto indexBase = Teuchos::as<GO>(epetraMap_.IndexBase()); if (epetraMap_.DistributedGlobal() || epetraMap_.Comm().NumProc() == Teuchos::OrdinalTraits<int>::one()) { Teuchos::Array<Tpetra_GO> indices(numElements); int *epetra_indices = epetraMap_.MyGlobalElements(); for(LO i=0; i < numElements; i++) indices[i] = epetra_indices[i]; const Tpetra::global_size_t computeGlobalElements = Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid(); return Teuchos::rcp(new Tpetra_Map(computeGlobalElements, indices, indexBase, commT_)); } else { return Teuchos::rcp(new Tpetra_Map(numElements, indexBase, commT_, Tpetra::LocallyReplicated)); } }
bool Vector2MATLAB( const Epetra_Vector & v, const Epetra_Map & Map) { int MyPID = Map.Comm().MyPID(); int NumProc = Map.Comm().NumProc(); int MyLength = v.MyLength(); int GlobalLength = v.GlobalLength(); // print out on cout if no filename is provided // write on file the dimension of the matrix if( MyPID == 0 ) cout << "v = zeros(" << GlobalLength << ")\n"; // get update list int * MyGlobalElements = Map.MyGlobalElements( ); int Row; for( int Proc=0 ; Proc<NumProc ; ++Proc ) { if( MyPID == Proc ) { cout << "% On proc " << Proc << ": "; cout << MyLength << " rows of "; cout << GlobalLength << " elements\n"; for( Row=0 ; Row<MyLength ; ++Row ) { cout << "b(" << MyGlobalElements[Row] << ") = " << v[Row] << ";\n"; } if( MyPID == NumProc-1 ) { cout << "% End of vector\n"; } } Map.Comm().Barrier(); } return true; } /* Vector2MATLAB */
// FIXME long long Epetra_Map Epetra_Util::Create_OneToOne_Map(const Epetra_Map& usermap, bool high_rank_proc_owns_shared) { //if usermap is already 1-to-1 then we'll just return a copy of it. if (usermap.IsOneToOne()) { Epetra_Map newmap(usermap); return(newmap); } int myPID = usermap.Comm().MyPID(); Epetra_Directory* directory = usermap.Comm().CreateDirectory(usermap); int numMyElems = usermap.NumMyElements(); const int* myElems = usermap.MyGlobalElements(); int* owner_procs = new int[numMyElems]; directory->GetDirectoryEntries(usermap, numMyElems, myElems, owner_procs, 0, 0, high_rank_proc_owns_shared); //we'll fill a list of map-elements which belong on this processor int* myOwnedElems = new int[numMyElems]; int numMyOwnedElems = 0; for(int i=0; i<numMyElems; ++i) { int GID = myElems[i]; int owner = owner_procs[i]; if (myPID == owner) { myOwnedElems[numMyOwnedElems++] = GID; } } Epetra_Map one_to_one_map(-1, numMyOwnedElems, myOwnedElems, usermap.IndexBase(), usermap.Comm()); delete [] myOwnedElems; delete [] owner_procs; delete directory; return(one_to_one_map); }
/* Computes the approximate Schur complement for the wide separator using guided probing*/ Teuchos::RCP<Epetra_CrsMatrix> computeSchur_GuidedProbing ( shylu_config *config, shylu_symbolic *ssym, // symbolic structure shylu_data *data, // numeric structure Epetra_Map *localDRowMap ) { int i; double relative_thres = config->relative_threshold; Epetra_CrsMatrix *G = ssym->G.getRawPtr(); Epetra_CrsMatrix *R = ssym->R.getRawPtr(); Epetra_LinearProblem *LP = ssym->LP.getRawPtr(); Amesos_BaseSolver *solver = ssym->Solver.getRawPtr(); Ifpack_Preconditioner *ifSolver = ssym->ifSolver.getRawPtr(); Epetra_CrsMatrix *C = ssym->C.getRawPtr(); // Need to create local G (block diagonal portion) , R, C // Get row map of G Epetra_Map CrMap = C->RowMap(); int *c_rows = CrMap.MyGlobalElements(); int *c_cols = (C->ColMap()).MyGlobalElements(); //int c_totalElems = CrMap.NumGlobalElements(); int c_localElems = CrMap.NumMyElements(); int c_localcolElems = (C->ColMap()).NumMyElements(); Epetra_Map GrMap = G->RowMap(); int *g_rows = GrMap.MyGlobalElements(); //int g_totalElems = GrMap.NumGlobalElements(); int g_localElems = GrMap.NumMyElements(); Epetra_Map RrMap = R->RowMap(); int *r_rows = RrMap.MyGlobalElements(); int *r_cols = (R->ColMap()).MyGlobalElements(); //int r_totalElems = RrMap.NumGlobalElements(); int r_localElems = RrMap.NumMyElements(); int r_localcolElems = (R->ColMap()).NumMyElements(); Epetra_SerialComm LComm; Epetra_Map C_localRMap (-1, c_localElems, c_rows, 0, LComm); Epetra_Map C_localCMap (-1, c_localcolElems, c_cols, 0, LComm); Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm); Epetra_Map R_localRMap (-1, r_localElems, r_rows, 0, LComm); Epetra_Map R_localCMap (-1, r_localcolElems, r_cols, 0, LComm); //cout << "#local rows" << g_localElems << "#non zero local cols" << c_localcolElems << endl; #ifdef DEBUG cout << "DEBUG MODE" << endl; int nrows = C->RowMap().NumMyElements(); assert(nrows == localDRowMap->NumGlobalElements()); int gids[nrows], gids1[nrows]; C_localRMap.MyGlobalElements(gids); localDRowMap->MyGlobalElements(gids1); cout << "Comparing R's domain map with D's row map" << endl; for (int i = 0; i < nrows; i++) { assert(gids[i] == gids1[i]); } #endif int nentries1, gid; // maxentries is the maximum of all three possible matrices as the arrays // are reused between the three int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries()); maxentries = max(maxentries, G->MaxNumEntries()); double *values1 = new double[maxentries]; double *values2 = new double[maxentries]; double *values3 = new double[maxentries]; int *indices1 = new int[maxentries]; int *indices2 = new int[maxentries]; int *indices3 = new int[maxentries]; //cout << "Creating local matrices" << endl; int err; Epetra_CrsMatrix localC(Copy, C_localRMap, C->MaxNumEntries(), false); for (i = 0; i < c_localElems ; i++) { gid = c_rows[i]; err = C->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); assert (err == 0); //if (nentries1 > 0) // TODO: Later //{ err = localC.InsertGlobalValues(gid, nentries1, values1, indices1); assert (err == 0); //} } localC.FillComplete(G_localRMap, C_localRMap); //cout << "Created local C matrix" << endl; Epetra_CrsMatrix localR(Copy, R_localRMap, R->MaxNumEntries(), false); for (i = 0; i < r_localElems ; i++) { gid = r_rows[i]; R->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); localR.InsertGlobalValues(gid, nentries1, values1, indices1); } localR.FillComplete(*localDRowMap, R_localRMap); //cout << "Created local R matrix" << endl; // Sbar - Approximate Schur complement Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix( Copy, GrMap, g_localElems)); // Include only the block diagonal elements of G in localG Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false); int cnt, scnt; for (i = 0; i < g_localElems ; i++) { gid = g_rows[i]; G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); cnt = 0; scnt = 0; for (int j = 0 ; j < nentries1 ; j++) { if (G->LRID(indices1[j]) != -1) { values2[cnt] = values1[j]; indices2[cnt++] = indices1[j]; } else { // Add it to Sbar immediately values3[scnt] = values1[j]; indices3[scnt++] = indices1[j]; } } localG.InsertGlobalValues(gid, cnt, values2, indices2); Sbar->InsertGlobalValues(gid, scnt, values3, indices3); } localG.FillComplete(); cout << "Created local G matrix" << endl; int nvectors = 16; ShyLU_Probing_Operator probeop(config, ssym, &localG, &localR, LP, solver, ifSolver, &localC, localDRowMap, nvectors); #ifdef DUMP_MATRICES //ostringstream fnamestr; //fnamestr << "localC" << C->Comm().MyPID() << ".mat"; //string Cfname = fnamestr.str(); //EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC); //Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm()); //EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg = //new EpetraExt::CrsMatrix_Reindex( defMapg ); //Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG ); //ReIdx_MatTransg->fwd(); //EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G); #endif //cout << " totalElems in Schur Complement" << totalElems << endl; //cout << myPID << " localElems" << localElems << endl; // **************** Two collectives here ********************* #ifdef TIMING_OUTPUT Teuchos::Time ftime("setup time"); #endif #ifdef TIMING_OUTPUT Teuchos::Time app_time("setup time"); #endif Teuchos::RCP<Epetra_CrsGraph> lSGraph = Teuchos::RCP<Epetra_CrsGraph> ( new Epetra_CrsGraph(Copy, G_localRMap, maxentries)); if (data->num_compute % config->reset_iter == 0) { int nentries; // size > maxentries as there could be fill // TODO: Currently the size of the two arrays can be one, Even if we switch // the loop below the size of the array required is nvectors. Fix it double *values = new double[g_localElems]; int *indices = new int[g_localElems]; double *vecvalues; int dropped = 0; double *maxvalue = new double[nvectors]; #ifdef TIMING_OUTPUT ftime.start(); #endif int findex = g_localElems / nvectors ; int cindex; // int mypid = C->Comm().MyPID(); // unused Epetra_MultiVector probevec(G_localRMap, nvectors); Epetra_MultiVector Scol(G_localRMap, nvectors); for (i = 0 ; i < findex*nvectors ; i+=nvectors) { probevec.PutScalar(0.0); // TODO: Move it out for (int k = 0; k < nvectors; k++) { cindex = k+i; // TODO: Can do better than this, just need to go to the column map // of C, there might be null columns in C // Not much of use for Shasta 2x2 .. Later. probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0); //if (mypid == 0) //cout << "Changing row to 1.0 " << g_rows[cindex] << endl; } #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif Scol.MaxValue(maxvalue); for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops { cindex = k+i; vecvalues = Scol[k]; //cout << "MAX" << maxvalue << endl; for (int j = 0 ; j < g_localElems ; j++) { nentries = 0; // inserting one entry in each row for now if (g_rows[cindex] == g_rows[j]) // diagonal entry { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[cindex]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres) { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[cindex]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else { if (vecvalues[j] != 0.0) { dropped++; //cout << "vecvalues[j]" << vecvalues[j] << // " max" << maxvalue[k] << endl; } } } } } probeop.ResetTempVectors(1); for ( ; i < g_localElems ; i++) { // TODO: Can move the next two decalarations outside the loop Epetra_MultiVector probevec(G_localRMap, 1); Epetra_MultiVector Scol(G_localRMap, 1); probevec.PutScalar(0.0); // TODO: Can do better than this, just need to go to the column map // of C, there might be null columns in C probevec.ReplaceGlobalValue(g_rows[i], 0, 1.0); #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif vecvalues = Scol[0]; Scol.MaxValue(maxvalue); //cout << "MAX" << maxvalue << endl; for (int j = 0 ; j < g_localElems ; j++) { nentries = 0; // inserting one entry in each row for now if (g_rows[i] == g_rows[j]) // diagonal entry { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[i]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres) { values[nentries] = vecvalues[j]; indices[nentries] = g_rows[i]; nentries++; err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); assert(err >= 0); err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices); assert(err >= 0); } else { if (vecvalues[j] != 0.0) dropped++; } } } #ifdef TIMING_OUTPUT ftime.stop(); cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl; ftime.reset(); #endif #ifdef TIMING_OUTPUT cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl; #endif probeop.PrintTimingInfo(); Sbar->FillComplete(); lSGraph->FillComplete(); data->localSbargraph = lSGraph; #ifdef DUMP_MATRICES Epetra_Map defMap2(-1, g_localElems, 0, C->Comm()); EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 = new EpetraExt::CrsMatrix_Reindex( defMap2 ); Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar ); ReIdx_MatTrans2->fwd(); EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S); #endif cout << "#dropped entries" << dropped << endl; delete[] values; delete[] indices; delete[] maxvalue; } else { if (((data->num_compute-1) % config->reset_iter) == 0) { // We recomputed the Schur complement with dropping for the last // compute. Reset the prober with the new orthogonal vectors for // the Sbar from the previous iteration. Teuchos::ParameterList pList; Teuchos::RCP<Isorropia::Epetra::Prober> gprober = Teuchos::RCP<Isorropia::Epetra::Prober> (new Isorropia::Epetra::Prober( data->localSbargraph.getRawPtr(), pList, false)); gprober->color(); data->guided_prober = gprober; } // Use the prober to probe the probeop for the sparsity pattern // add that to Sbar and call Fill complete int nvectors = data->guided_prober->getNumOrthogonalVectors(); cout << "Number of Orthogonal Vectors for guided probing" << nvectors << endl; probeop.ResetTempVectors(nvectors); Teuchos::RCP<Epetra_CrsMatrix> blockdiag_Sbar = data->guided_prober->probe(probeop); int maxentries = blockdiag_Sbar->GlobalMaxNumEntries(); int *indices = new int[maxentries]; double *values = new double[maxentries]; int numentries; for (int i = 0; i < blockdiag_Sbar->NumGlobalRows() ; i++) { int gid = blockdiag_Sbar->GRID(i); blockdiag_Sbar->ExtractGlobalRowCopy(gid, maxentries, numentries, values, indices); Sbar->InsertGlobalValues(gid, numentries, values, indices); } Sbar->FillComplete(); delete[] indices; delete[] values; } delete[] values1; delete[] indices1; delete[] values2; delete[] indices2; delete[] values3; delete[] indices3; return Sbar; }
/* Apply an identity matrix to the Schur complement operator. Drop the entries entries using a relative threshold. Assemble the result in a Crs Matrix which will be our approximate Schur complement. */ Teuchos::RCP<Epetra_CrsMatrix> computeApproxSchur(shylu_config *config, shylu_symbolic *sym, Epetra_CrsMatrix *G, Epetra_CrsMatrix *R, Epetra_LinearProblem *LP, Amesos_BaseSolver *solver, Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C, Epetra_Map *localDRowMap) { double relative_thres = config->relative_threshold; int nvectors = 16; ShyLU_Probing_Operator probeop(config, sym, G, R, LP, solver, ifSolver, C, localDRowMap, nvectors); // Get row map Epetra_Map rMap = G->RowMap(); int *rows = rMap.MyGlobalElements(); int totalElems = rMap.NumGlobalElements(); int localElems = rMap.NumMyElements(); //cout << " totalElems in Schur Complement" << totalElems << endl; //cout << myPID << " localElems" << localElems << endl; // **************** Two collectives here ********************* #ifdef TIMING_OUTPUT Teuchos::Time ftime("setup time"); ftime.start(); #endif int prefixSum; G->Comm().ScanSum(&localElems, &prefixSum, 1); //cout << " prefixSum" << prefixSum << endl; // Start the index in prefixSum-localElems int *mySGID = new int[totalElems]; // vector of size Schur complement ! int *allSGID = new int[totalElems]; // vector of size Schur complement ! int i, j; for (i = 0, j = 0; i < totalElems ; i++) { if (i >= prefixSum - localElems && i < prefixSum) { mySGID[i] = rows[j]; j++; } else { mySGID[i] = 0; } allSGID[i] = 0; } C->Comm().SumAll(mySGID, allSGID, totalElems); #ifdef TIMING_OUTPUT ftime.stop(); cout << "Time to Compute RowIDS" << ftime.totalElapsedTime() << endl; ftime.reset(); #endif // Now everyone knows the GIDs in the Schur complement //cout << rMap << endl; j = 0; Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix( Copy, rMap, localElems)); int nentries; double *values = new double[localElems]; // Need to adjust this for more int *indices = new int[localElems]; // than one vector double *vecvalues; int dropped = 0; double *maxvalue = new double[nvectors]; #ifdef TIMING_OUTPUT ftime.start(); #endif #ifdef TIMING_OUTPUT Teuchos::Time app_time("Apply time"); #endif int findex = totalElems / nvectors ; for (i = 0 ; i < findex*nvectors ; i+=nvectors) { Epetra_MultiVector probevec(rMap, nvectors); Epetra_MultiVector Scol(rMap, nvectors); probevec.PutScalar(0.0); int cindex; for (int k = 0; k < nvectors; k++) { cindex = k+i; if (cindex >= prefixSum - localElems && cindex < prefixSum) { probevec.ReplaceGlobalValue(allSGID[cindex], k, 1.0); } } #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif Scol.MaxValue(maxvalue); for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops { cindex = k+i; vecvalues = Scol[k]; //cout << "MAX" << maxvalue << endl; for (j = 0 ; j < localElems ; j++) { nentries = 0; // inserting one entry in each row for now if (allSGID[cindex] == rows[j]) // diagonal entry { values[nentries] = vecvalues[j]; indices[nentries] = allSGID[cindex]; nentries++; Sbar->InsertGlobalValues(rows[j], nentries, values, indices); } else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres) { values[nentries] = vecvalues[j]; indices[nentries] = allSGID[cindex]; nentries++; Sbar->InsertGlobalValues(rows[j], nentries, values, indices); } else { if (vecvalues[j] != 0.0) dropped++; } } } } probeop.ResetTempVectors(1); for ( ; i < totalElems ; i++) { Epetra_MultiVector probevec(rMap, 1); // TODO: Try doing more than one Epetra_MultiVector Scol(rMap, 1); // vector at a time probevec.PutScalar(0.0); if (i >= prefixSum - localElems && i < prefixSum) { probevec.ReplaceGlobalValue(allSGID[i], 0, 1.0); } #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif vecvalues = Scol[0]; Scol.MaxValue(maxvalue); //cout << "MAX" << maxvalue << endl; for (j = 0 ; j < localElems ; j++) { nentries = 0; // inserting one entry in each row for now if (allSGID[i] == rows[j]) // diagonal entry { values[nentries] = vecvalues[j]; indices[nentries] = allSGID[i]; nentries++; Sbar->InsertGlobalValues(rows[j], nentries, values, indices); } else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres) { values[nentries] = vecvalues[j]; indices[nentries] = allSGID[i]; nentries++; Sbar->InsertGlobalValues(rows[j], nentries, values, indices); } else { if (vecvalues[j] != 0.0) dropped++; } } } #ifdef TIMING_OUTPUT ftime.stop(); cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl; ftime.reset(); #endif #ifdef TIMING_OUTPUT cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl; #endif Sbar->FillComplete(); cout << "#dropped entries" << dropped << endl; delete[] allSGID; delete[] mySGID; delete[] values; delete[] indices; delete[] maxvalue; return Sbar; }
int CreateTridi(Epetra_CrsMatrix& A) { Epetra_Map Map = A.RowMap(); int NumMyElements = Map.NumMyElements(); int NumGlobalElements = Map.NumGlobalElements(); int * MyGlobalElements = new int[NumMyElements]; Map.MyGlobalElements(MyGlobalElements); // Add rows one-at-a-time // Need some vectors to help // Off diagonal Values will always be -1 double *Values = new double[3]; int *Indices = new int[3]; int NumEntries; for (int i=0; i<NumMyElements; i++) { if (MyGlobalElements[i]==0) { Indices[0] = 0; Indices[1] = 1; Values[0] = 2.0; Values[1] = -1.0; NumEntries = 2; } else if (MyGlobalElements[i] == NumGlobalElements-1) { Indices[0] = NumGlobalElements-1; Indices[1] = NumGlobalElements-2; Values[0] = 2.0; Values[1] = -1.0; NumEntries = 2; } else { Indices[0] = MyGlobalElements[i]-1; Indices[1] = MyGlobalElements[i]; Indices[2] = MyGlobalElements[i]+1; Values[0] = -1.0; Values[1] = 2.0; Values[2] = -1.0; NumEntries = 3; } assert(A.InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices)==0); // Put in the diagonal entry // assert(A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i])==0); } // Finish up assert(A.FillComplete()==0); delete[] MyGlobalElements; delete[] Values; delete[] Indices; return 0; }
/* Computes the approximate Schur complement for the wide separator */ Teuchos::RCP<Epetra_CrsMatrix> computeApproxWideSchur(shylu_config *config, shylu_symbolic *ssym, // symbolic structure Epetra_CrsMatrix *G, Epetra_CrsMatrix *R, Epetra_LinearProblem *LP, Amesos_BaseSolver *solver, Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C, Epetra_Map *localDRowMap) { int i; double relative_thres = config->relative_threshold; // Need to create local G (block diagonal portion) , R, C // Get row map of G //Epetra_Map CrMap = C->RowMap(); //int *c_rows = CrMap.MyGlobalElements(); //int *c_cols = (C->ColMap()).MyGlobalElements(); //int c_totalElems = CrMap.NumGlobalElements(); //int c_localElems = CrMap.NumMyElements(); //int c_localcolElems = (C->ColMap()).NumMyElements(); Epetra_Map GrMap = G->RowMap(); int *g_rows = GrMap.MyGlobalElements(); //int g_totalElems = GrMap.NumGlobalElements(); int g_localElems = GrMap.NumMyElements(); //Epetra_Map RrMap = R->RowMap(); //int *r_rows = RrMap.MyGlobalElements(); //int *r_cols = (R->ColMap()).MyGlobalElements(); //int r_totalElems = RrMap.NumGlobalElements(); //int r_localElems = RrMap.NumMyElements(); //int r_localcolElems = (R->ColMap()).NumMyElements(); Epetra_SerialComm LComm; Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm); int nentries1, gid; // maxentries is the maximum of all three possible matrices as the arrays // are reused between the three int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries()); maxentries = max(maxentries, G->MaxNumEntries()); double *values1 = new double[maxentries]; double *values2 = new double[maxentries]; double *values3 = new double[maxentries]; int *indices1 = new int[maxentries]; int *indices2 = new int[maxentries]; int *indices3 = new int[maxentries]; // Sbar - Approximate Schur complement Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix( Copy, GrMap, g_localElems)); // Include only the block diagonal elements of G in localG Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false); int cnt, scnt; for (i = 0; i < g_localElems ; i++) { gid = g_rows[i]; G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1); cnt = 0; scnt = 0; for (int j = 0 ; j < nentries1 ; j++) { if (G->LRID(indices1[j]) != -1) { values2[cnt] = values1[j]; indices2[cnt++] = indices1[j]; } else { // Add it to Sbar immediately values3[scnt] = values1[j]; indices3[scnt++] = indices1[j]; } } localG.InsertGlobalValues(gid, cnt, values2, indices2); Sbar->InsertGlobalValues(gid, scnt, values3, indices3); } localG.FillComplete(); //cout << "Created local G matrix" << endl; int nvectors = 16; /*ShyLU_Probing_Operator probeop(&localG, &localR, LP, solver, &localC, localDRowMap, nvectors);*/ ShyLU_Local_Schur_Operator probeop(config, ssym, &localG, R, LP, solver, ifSolver, C, localDRowMap, nvectors); #ifdef DUMP_MATRICES //ostringstream fnamestr; //fnamestr << "localC" << C->Comm().MyPID() << ".mat"; //string Cfname = fnamestr.str(); //EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC); //Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm()); //EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg = //new EpetraExt::CrsMatrix_Reindex( defMapg ); //Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG ); //ReIdx_MatTransg->fwd(); //EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G); #endif //cout << " totalElems in Schur Complement" << totalElems << endl; //cout << myPID << " localElems" << localElems << endl; // **************** Two collectives here ********************* #ifdef TIMING_OUTPUT Teuchos::Time ftime("setup time"); ftime.start(); #endif #ifdef TIMING_OUTPUT Teuchos::Time app_time("setup time"); #endif int nentries; // size > maxentries as there could be fill // TODO: Currently the size of the two arrays can be one, Even if we switch // the loop below the size of the array required is nvectors. Fix it double *values = new double[nvectors]; int *indices = new int[nvectors]; double *vecvalues; #ifdef SHYLU_DEBUG // mfh 25 May 2015: Don't declare this variable if it's not used. // It's only used if SHYLU_DEBUG is defined. int dropped = 0; #endif // SHYLU_DEBUG double *maxvalue = new double[nvectors]; #ifdef TIMING_OUTPUT ftime.start(); #endif int findex = g_localElems / nvectors ; int cindex; // int mypid = C->Comm().MyPID(); // unused Epetra_MultiVector probevec (G_localRMap, nvectors); Epetra_MultiVector Scol (G_localRMap, nvectors); probevec.PutScalar(0.0); for (i = 0 ; i < findex*nvectors ; i+=nvectors) { // Set the probevec to find block columns of S. for (int k = 0; k < nvectors; k++) { cindex = k+i; // TODO: Can do better than this, just need to go to the column map // of C, there might be null columns in C probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0); //if (mypid == 0) //cout << "Changing row to 1.0 " << g_rows[cindex] << endl; } #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec, Scol); #ifdef TIMING_OUTPUT app_time.stop(); #endif // Reset the probevec to all zeros. for (int k = 0; k < nvectors; k++) { cindex = k+i; probevec.ReplaceGlobalValue(g_rows[cindex], k, 0.0); } Scol.MaxValue(maxvalue); nentries = 0; for (int j = 0 ; j < g_localElems ; j++) { for (int k = 0; k < nvectors; k++) { cindex = k+i; vecvalues = Scol[k]; if ((g_rows[cindex] == g_rows[j]) || (abs(vecvalues[j]/maxvalue[k]) > relative_thres)) // diagonal entry or large entry. { values[nentries] = vecvalues[j]; indices[nentries++] = g_rows[cindex]; } #ifdef SHYLU_DEBUG else if (vecvalues[j] != 0.0) { dropped++; } #endif // SHYLU_DEBUG } Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); nentries = 0; } } if (i < g_localElems) { nvectors = g_localElems - i; probeop.ResetTempVectors(nvectors); Epetra_MultiVector probevec1 (G_localRMap, nvectors); Epetra_MultiVector Scol1 (G_localRMap, nvectors); probevec1.PutScalar(0.0); for (int k = 0; k < nvectors; k++) { cindex = k+i; // TODO: Can do better than this, just need to go to the column map // of C, there might be null columns in C probevec1.ReplaceGlobalValue(g_rows[cindex], k, 1.0); } #ifdef TIMING_OUTPUT app_time.start(); #endif probeop.Apply(probevec1, Scol1); #ifdef TIMING_OUTPUT app_time.stop(); #endif Scol1.MaxValue(maxvalue); nentries = 0; for (int j = 0 ; j < g_localElems ; j++) { //cout << "MAX" << maxvalue << endl; for (int k = 0; k < nvectors; k++) { cindex = k+i; vecvalues = Scol1[k]; //nentries = 0; // inserting one entry in each row for now if ((g_rows[cindex] == g_rows[j]) || (abs(vecvalues[j]/maxvalue[k]) > relative_thres)) // diagonal entry or large entry. { values[nentries] = vecvalues[j]; indices[nentries++] = g_rows[cindex]; } #ifdef SHYLU_DEBUG else if (vecvalues[j] != 0.0) { dropped++; } #endif // SHYLU_DEBUG } Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices); nentries = 0; } } #ifdef TIMING_OUTPUT ftime.stop(); cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl; ftime.reset(); cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl; probeop.PrintTimingInfo(); #endif Sbar->FillComplete(); #ifdef DUMP_MATRICES Epetra_Map defMap2(-1, g_localElems, 0, C->Comm()); EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 = new EpetraExt::CrsMatrix_Reindex( defMap2 ); Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar ); ReIdx_MatTrans2->fwd(); EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S); #endif #ifdef SHYLU_DEBUG cout << "#dropped entries" << dropped << endl; #endif delete[] values; delete[] indices; delete[] values1; delete[] indices1; delete[] values2; delete[] indices2; delete[] values3; delete[] indices3; delete[] maxvalue; return Sbar; }
void LOCA::Epetra::AugmentedOp::buildExtendedMap(const Epetra_BlockMap& uMap, Epetra_Map*& eMapPtr, bool buildImporter, bool haveParam) { Epetra_BlockMap& nonconstUnderlyingMap = const_cast<Epetra_BlockMap&>(uMap); // Convert underlying map to point map if necessary Epetra_Map* uPointMapPtr = dynamic_cast<Epetra_Map*>(&nonconstUnderlyingMap); bool allocatedPointMap = false; if (uPointMapPtr == NULL) { allocatedPointMap = true; blockMap2PointMap(uMap, uPointMapPtr); } int max_gid = uPointMapPtr->MaxAllGID(); int num_global_elements = uPointMapPtr->NumGlobalElements(); int num_my_elements = uPointMapPtr->NumMyElements(); int *global_elements = uPointMapPtr->MyGlobalElements(); const Epetra_Comm& comm = uPointMapPtr->Comm(); int index_base = uPointMapPtr->IndexBase(); int ext_num_global_elements; int ext_num_my_elements; int *ext_global_elements; // Compute number of extended global elements if (buildImporter) ext_num_global_elements = num_global_elements + numConstraints*comm.NumProc(); else ext_num_global_elements = num_global_elements + numConstraints; // Compute number of extended local elements if (buildImporter || haveParam) ext_num_my_elements = num_my_elements + numConstraints; else ext_num_my_elements = num_my_elements; // Allocate extended global elements array ext_global_elements = new int[ext_num_my_elements]; // Set extended global elements for (int i=0; i<num_my_elements; i++) { ext_global_elements[i] = global_elements[i]; } if (buildImporter || haveParam) for (int i=0; i<numConstraints; i++) ext_global_elements[num_my_elements+i] = max_gid + 1 + i; // Create extended point map eMapPtr = new Epetra_Map(ext_num_global_elements, ext_num_my_elements, ext_global_elements, index_base, comm); // Free global elements array delete [] ext_global_elements; if (allocatedPointMap) delete uPointMapPtr; }
int Amesos_TestSolver( Epetra_Comm &Comm, char *matrix_file, SparseSolverType SparseSolver, bool transpose, int special, AMESOS_MatrixType matrix_type ) { Epetra_Map * readMap; Epetra_CrsMatrix * readA; Epetra_Vector * readx; Epetra_Vector * readb; Epetra_Vector * readxexact; std::string FileName = matrix_file ; int FN_Size = FileName.size() ; std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size ); std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size ); bool NonContiguousMap = false; if ( LastFiveBytes == ".triU" ) { // Call routine to read in unsymmetric Triplet matrix NonContiguousMap = true; EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, false, Comm, readMap, readA, readx, readb, readxexact, NonContiguousMap ) ); } else { if ( LastFiveBytes == ".triS" ) { NonContiguousMap = true; // Call routine to read in symmetric Triplet matrix EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, true, Comm, readMap, readA, readx, readb, readxexact, NonContiguousMap ) ); } else { if ( LastFourBytes == ".mtx" ) { EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( matrix_file, Comm, readMap, readA, readx, readb, readxexact) ); } else { // Call routine to read in HB problem Trilinos_Util_ReadHb2Epetra( matrix_file, Comm, readMap, readA, readx, readb, readxexact) ; } } } Epetra_CrsMatrix transposeA(Copy, *readMap, 0); Epetra_CrsMatrix *serialA ; if ( transpose ) { assert( CrsMatrixTranspose( readA, &transposeA ) == 0 ); serialA = &transposeA ; } else { serialA = readA ; } Epetra_RowMatrix * passA = 0; Epetra_Vector * passx = 0; Epetra_Vector * passb = 0; Epetra_Vector * passxexact = 0; Epetra_Vector * passresid = 0; Epetra_Vector * passtmp = 0; // Create uniform distributed map Epetra_Map map(readMap->NumGlobalElements(), 0, Comm); Epetra_Map* map_; if( NonContiguousMap ) { // // map gives us NumMyElements and MyFirstElement; // int NumGlobalElements = readMap->NumGlobalElements(); int NumMyElements = map.NumMyElements(); int MyFirstElement = map.MinMyGID(); std::vector<int> MapMap_( NumGlobalElements ); readMap->MyGlobalElements( &MapMap_[0] ) ; Comm.Broadcast( &MapMap_[0], NumGlobalElements, 0 ) ; map_ = new Epetra_Map( NumGlobalElements, NumMyElements, &MapMap_[MyFirstElement], 0, Comm); } else { map_ = new Epetra_Map( map ) ; } Epetra_CrsMatrix A(Copy, *map_, 0); const Epetra_Map &OriginalMap = serialA->RowMatrixRowMap() ; assert( OriginalMap.SameAs(*readMap) ); Epetra_Export exporter(OriginalMap, *map_); Epetra_Export exporter2(OriginalMap, *map_); Epetra_Export MatrixExporter(OriginalMap, *map_); Epetra_CrsMatrix AwithDiag(Copy, *map_, 0); Epetra_Vector x(*map_); Epetra_Vector b(*map_); Epetra_Vector xexact(*map_); Epetra_Vector resid(*map_); Epetra_Vector readresid(*readMap); Epetra_Vector tmp(*map_); Epetra_Vector readtmp(*readMap); // Epetra_Vector xcomp(*map_); // X as computed by the solver bool distribute_matrix = ( matrix_type == AMESOS_Distributed ) ; if ( distribute_matrix ) { // Create Exporter to distribute read-in matrix and vectors // // Initialize x, b and xexact to the values read in from the file // x.Export(*readx, exporter, Add); b.Export(*readb, exporter, Add); xexact.Export(*readxexact, exporter, Add); Comm.Barrier(); A.Export(*serialA, exporter, Add); assert(A.FillComplete()==0); Comm.Barrier(); passA = &A; passx = &x; passb = &b; passxexact = &xexact; passresid = &resid; passtmp = &tmp; } else { passA = serialA; passx = readx; passb = readb; passxexact = readxexact; passresid = &readresid; passtmp = &readtmp; } Epetra_MultiVector CopyB( *passb ) ; double Anorm = passA->NormInf() ; SparseDirectTimingVars::SS_Result.Set_Anorm(Anorm) ; Epetra_LinearProblem Problem( (Epetra_RowMatrix *) passA, (Epetra_MultiVector *) passx, (Epetra_MultiVector *) passb ); for ( int i = 0; i < 1+special ; i++ ) { Epetra_Time TotalTime( Comm ) ; if ( false ) { // TEST_UMFPACK is never set by configure #ifdef HAVE_AMESOS_SUPERLUDIST } else if ( SparseSolver == SUPERLUDIST ) { Teuchos::ParameterList ParamList ; ParamList.set( "MaxProcs", -3 ); Amesos_Superludist A_Superludist( Problem ) ; //ParamList.set( "Redistribute", true ); //ParamList.set( "AddZeroToDiag", true ); Teuchos::ParameterList& SuperludistParams = ParamList.sublist("Superludist") ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_Superludist.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_Superludist.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_Superludist.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_Superludist.NumericFactorization( ) ); EPETRA_CHK_ERR( A_Superludist.Solve( ) ); #endif #ifdef HAVE_AMESOS_DSCPACK } else if ( SparseSolver == DSCPACK ) { Teuchos::ParameterList ParamList ; ParamList.set( "MaxProcs", -3 ); Amesos_Dscpack A_dscpack( Problem ) ; EPETRA_CHK_ERR( A_dscpack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_dscpack.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_dscpack.NumericFactorization( ) ); EPETRA_CHK_ERR( A_dscpack.Solve( ) ); #endif #ifdef HAVE_AMESOS_SCALAPACK } else if ( SparseSolver == SCALAPACK ) { Teuchos::ParameterList ParamList ; Amesos_Scalapack A_scalapack( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_scalapack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_scalapack.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_scalapack.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_scalapack.NumericFactorization( ) ); EPETRA_CHK_ERR( A_scalapack.Solve( ) ); #endif #ifdef HAVE_AMESOS_TAUCS } else if ( SparseSolver == TAUCS ) { Teuchos::ParameterList ParamList ; Amesos_Taucs A_taucs( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_taucs.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_taucs.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_taucs.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_taucs.NumericFactorization( ) ); EPETRA_CHK_ERR( A_taucs.Solve( ) ); #endif #ifdef HAVE_AMESOS_PARDISO } else if ( SparseSolver == PARDISO ) { Teuchos::ParameterList ParamList ; Amesos_Pardiso A_pardiso( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_pardiso.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_pardiso.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_pardiso.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_pardiso.NumericFactorization( ) ); EPETRA_CHK_ERR( A_pardiso.Solve( ) ); #endif #ifdef HAVE_AMESOS_PARAKLETE } else if ( SparseSolver == PARAKLETE ) { Teuchos::ParameterList ParamList ; Amesos_Paraklete A_paraklete( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_paraklete.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_paraklete.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_paraklete.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_paraklete.NumericFactorization( ) ); EPETRA_CHK_ERR( A_paraklete.Solve( ) ); #endif #ifdef HAVE_AMESOS_MUMPS } else if ( SparseSolver == MUMPS ) { Teuchos::ParameterList ParamList ; Amesos_Mumps A_mumps( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_mumps.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_mumps.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_mumps.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_mumps.NumericFactorization( ) ); EPETRA_CHK_ERR( A_mumps.Solve( ) ); #endif #ifdef HAVE_AMESOS_SUPERLU } else if ( SparseSolver == SUPERLU ) { Teuchos::ParameterList ParamList ; Amesos_Superlu A_superlu( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_superlu.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_superlu.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_superlu.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_superlu.NumericFactorization( ) ); EPETRA_CHK_ERR( A_superlu.Solve( ) ); #endif #ifdef HAVE_AMESOS_LAPACK } else if ( SparseSolver == LAPACK ) { Teuchos::ParameterList ParamList ; Amesos_Lapack A_lapack( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_lapack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_lapack.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_lapack.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_lapack.NumericFactorization( ) ); EPETRA_CHK_ERR( A_lapack.Solve( ) ); #endif #ifdef HAVE_AMESOS_UMFPACK } else if ( SparseSolver == UMFPACK ) { Teuchos::ParameterList ParamList ; Amesos_Umfpack A_umfpack( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( A_umfpack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( A_umfpack.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( A_umfpack.SymbolicFactorization( ) ); EPETRA_CHK_ERR( A_umfpack.NumericFactorization( ) ); EPETRA_CHK_ERR( A_umfpack.Solve( ) ); #endif #ifdef HAVE_AMESOS_KLU } else if ( SparseSolver == KLU ) { using namespace Teuchos; Amesos_Time AT; int setupTimePtr = -1, symTimePtr = -1, numTimePtr = -1, refacTimePtr = -1, solveTimePtr = -1; AT.CreateTimer(Comm, 2); AT.ResetTimer(0); Teuchos::ParameterList ParamList ; // ParamList.set("OutputLevel",2); Amesos_Klu A_klu( Problem ); ParamList.set( "MaxProcs", -3 ); ParamList.set( "TrustMe", false ); // ParamList.set( "Refactorize", true ); EPETRA_CHK_ERR( A_klu.SetParameters( ParamList ) ) ; EPETRA_CHK_ERR( A_klu.SetUseTranspose( transpose ) ); setupTimePtr = AT.AddTime("Setup", setupTimePtr, 0); EPETRA_CHK_ERR( A_klu.SymbolicFactorization( ) ); symTimePtr = AT.AddTime("Symbolic", symTimePtr, 0); EPETRA_CHK_ERR( A_klu.NumericFactorization( ) ); numTimePtr = AT.AddTime("Numeric", numTimePtr, 0); EPETRA_CHK_ERR( A_klu.NumericFactorization( ) ); refacTimePtr = AT.AddTime("Refactor", refacTimePtr, 0); // for ( int i=0; i<100000 ; i++ ) EPETRA_CHK_ERR( A_klu.Solve( ) ); solveTimePtr = AT.AddTime("Solve", solveTimePtr, 0); double SetupTime = AT.GetTime(setupTimePtr); double SymbolicTime = AT.GetTime(symTimePtr); double NumericTime = AT.GetTime(numTimePtr); double RefactorTime = AT.GetTime(refacTimePtr); double SolveTime = AT.GetTime(solveTimePtr); std::cout << __FILE__ << "::" << __LINE__ << " SetupTime = " << SetupTime << std::endl ; std::cout << __FILE__ << "::" << __LINE__ << " SymbolicTime = " << SymbolicTime - SetupTime << std::endl ; std::cout << __FILE__ << "::" << __LINE__ << " NumericTime = " << NumericTime - SymbolicTime<< std::endl ; std::cout << __FILE__ << "::" << __LINE__ << " RefactorTime = " << RefactorTime - NumericTime << std::endl ; std::cout << __FILE__ << "::" << __LINE__ << " SolveTime = " << SolveTime - RefactorTime << std::endl ; #endif } else { SparseDirectTimingVars::log_file << "Solver not implemented yet" << std::endl ; std::cerr << "\n\n#################### Requested solver not available on this platform ##################### ATS\n" << std::endl ; std::cout << " SparseSolver = " << SparseSolver << std::endl ; std::cerr << " SparseSolver = " << SparseSolver << std::endl ; } SparseDirectTimingVars::SS_Result.Set_Total_Time( TotalTime.ElapsedTime() ); } // end for (int i=0; i<special; i++ ) // // Compute the error = norm(xcomp - xexact ) // double error; passresid->Update(1.0, *passx, -1.0, *passxexact, 0.0); passresid->Norm2(&error); SparseDirectTimingVars::SS_Result.Set_Error(error) ; // passxexact->Norm2(&error ) ; // passx->Norm2(&error ) ; // // Compute the residual = norm(Ax - b) // double residual ; passA->Multiply( transpose, *passx, *passtmp); passresid->Update(1.0, *passtmp, -1.0, *passb, 0.0); // passresid->Update(1.0, *passtmp, -1.0, CopyB, 0.0); passresid->Norm2(&residual); SparseDirectTimingVars::SS_Result.Set_Residual(residual) ; double bnorm; passb->Norm2( &bnorm ) ; SparseDirectTimingVars::SS_Result.Set_Bnorm(bnorm) ; double xnorm; passx->Norm2( &xnorm ) ; SparseDirectTimingVars::SS_Result.Set_Xnorm(xnorm) ; delete readA; delete readx; delete readb; delete readxexact; delete readMap; delete map_; Comm.Barrier(); return 0; }
/* Find the DBBD form */ int shylu_symbolic_factor ( Epetra_CrsMatrix *A, // i/p: A matrix shylu_symbolic *ssym, // symbolic structure shylu_data *data, // numeric structure, TODO: Required ? shylu_config *config // i/p: library configuration ) { #ifdef TIMING_OUTPUT Teuchos::Time symtime("symbolic time"); symtime.start(); #endif int myPID = A->Comm().MyPID(); int n = A->NumGlobalRows(); int Dnr; int Snr; int *DRowElems; int *SRowElems; int sym = config->sym; checkMaps(A); // Get column map Epetra_Map AColMap = A->ColMap(); int ncols = AColMap.NumMyElements(); int *cols = AColMap.MyGlobalElements(); // Get row map Epetra_Map ARowMap = A->RowMap(); int nrows = ARowMap.NumMyElements(); int *rows = ARowMap.MyGlobalElements(); // Find all columns in this proc int *gvals = new int[n]; // vector of size n, not ncols ! // gvals[local cols] = 1, gvals[shared cols] > 1. int SNumGlobalCols; findLocalColumns(A, gvals, SNumGlobalCols); // See if you can shrink the separator by assigning more rows/columns to // the block diagonals // TODO: This is because of a bug in coloring remove the if once that is // fixed //if (config->schurApproxMethod == 2) if (config->sep_type == 2) findNarrowSeparator(A, gvals); // 3. Assemble diagonal block and the border in convenient form [ /* In each processor, we have (in a permuted form) * | D_i C_i | * | R_i S_i | * D_i - diagonal block, C_i - Column Separator, R_i - Row separator * S_i - A22 block corresponding to Schur complement part of A * Assemble all four blocks in local matrices. */ ostringstream ssmsg1; ssmsg1 << "PID =" << myPID << " "; string msg = ssmsg1.str(); ssmsg1.clear(); ssmsg1.str(""); // Find #cols in each block int Dnc = 0; // #cols in diagonal block int Snc = 0; // #cols in the col. separator /* Looping on cols will work only for wide separator * as for narrow sep there will be some sep cols with gvals[col] ==1 * */ /*for (int i = 0; i < ncols ; i++) { if (gvals[cols[i]] == 1) Dnc++; else Snc++; } // Find #rows in each block Dnr = Dnc; // #rows in square diagonal block Snr = nrows - Dnr; // #rows in the row separator*/ // Find #rows in each block Dnr = 0; Snr = 0; for (int i = 0; i < nrows ; i++) { if (gvals[rows[i]] == 1) Dnr++; else Snr++; } Dnc = Dnr; // TODO: Snc is no longer useful, should remove it for (int i = 0; i < ncols ; i++) { if (gvals[cols[i]] != 1) Snc++; } assert(Snc >= 0); // TODO : The above assignment may not be correct in the unsymetric case ////config->dm.print(2, msg + " Mycols="); cout << msg << " Mycols="<< ncols << "Myrows ="<< nrows << endl; cout << msg << " #rows and #cols in diagonal blk ="<< Dnr << endl; cout << msg << " #columns in S ="<< Snc << endl; cout << msg << " #rows in S ="<< Snr << endl; ostringstream pidstr; pidstr << myPID ; // Create a row map for the D and S blocks [ DRowElems = new int[Dnr]; SRowElems = new int[Snr]; int gid; // Assemble row ids in two arrays (for D and R blocks) if (sym) { findBlockElems(A, nrows, rows, gvals, Dnr, DRowElems, Snr, SRowElems, "D"+pidstr.str()+"Rows", "S"+pidstr.str()+"Rows", false) ; } else { // SRowElems are not known until factorization, TODO assert(0 == 1); } data->Dnr = Dnr; data->Snr = Snr; data->Dnc = Dnc; data->DRowElems = DRowElems; data->SRowElems = SRowElems; // Create a column map for the D and S blocks [ int *DColElems = new int[Dnc]; // Elems in column map of D int *SColElems = new int[Snc]; // Elems in column map of C TODO: Unused // Assemble column ids in two arrays (for D and C blocks) findBlockElems(A, ncols, cols, gvals, Dnc, DColElems, Snc, SColElems, "D"+pidstr.str()+"Cols", "S"+pidstr.str()+"Cols", true) ; data->DColElems = DColElems; data->gvals = gvals; for (int i = 0; i < Snr; i++) { // Epetra guarentees columns corresponding to local rows will be first // in the column map. assert(SRowElems[i] == SColElems[i]); } // ] /*--Create the Epetra Matrices with the maps (does not insert values) --- */ create_matrices(A, ssym, data, config); /*--Extract the Epetra Matrices and call fillComplete --- */ extract_matrices(A, ssym, data, config, true); delete[] SColElems; Amesos Factory; const char* SolverType = config->diagonalBlockSolver.c_str(); bool IsAvailable = Factory.Query(SolverType); assert(IsAvailable == true); Teuchos::RCP<Epetra_LinearProblem> LP = Teuchos::RCP<Epetra_LinearProblem> (new Epetra_LinearProblem()); LP->SetOperator((ssym->D).getRawPtr()); //LP->SetOperator((ssym->DT).getRawPtr()); // for transpose // Create temp vectors ssym->Dlhs = Teuchos::RCP<Epetra_MultiVector> (new Epetra_MultiVector(ssym->D->RowMap(), 16)); ssym->Drhs = Teuchos::RCP<Epetra_MultiVector> (new Epetra_MultiVector(ssym->D->RowMap(), 16)); ssym->Gvec = Teuchos::RCP<Epetra_MultiVector> (new Epetra_MultiVector(ssym->G->RowMap(), 16)); LP->SetRHS(ssym->Drhs.getRawPtr()); LP->SetLHS(ssym->Dlhs.getRawPtr()); ssym->ReIdx_LP = Teuchos::RCP< EpetraExt::ViewTransform<Epetra_LinearProblem> > (new EpetraExt::LinearProblem_Reindex2(0)); ssym->LP = Teuchos::RCP<Epetra_LinearProblem>(&((*(ssym->ReIdx_LP))(*LP)), false); Teuchos::RCP<Amesos_BaseSolver> Solver = Teuchos::RCP<Amesos_BaseSolver> (Factory.Create(SolverType, *(ssym->LP))); //config->dm.print(5, "Created the diagonal solver"); #ifdef TIMING_OUTPUT Teuchos::Time ftime("setup time"); ftime.start(); #endif //Solver->SetUseTranspose(true); // for transpose Teuchos::ParameterList aList; aList.set("TrustMe", true); Solver->SetParameters(aList); Solver->SymbolicFactorization(); //config->dm.print(3, "Symbolic Factorization done"); #ifdef TIMING_OUTPUT ftime.stop(); cout << "Symbolic Factorization Time" << ftime.totalElapsedTime() << endl; ftime.reset(); #endif ssym->OrigLP = LP; //ssym->LP = LP; ssym->Solver = Solver; if (config->schurApproxMethod == 1) { Teuchos::ParameterList pList; Teuchos::RCP<Isorropia::Epetra::Prober> prober = Teuchos::RCP<Isorropia::Epetra::Prober> (new Isorropia::Epetra::Prober((ssym->Sg).getRawPtr(), pList, false)); //config->dm.print(3, "Doing Coloring"); #ifdef TIMING_OUTPUT ftime.start(); #endif prober->color(); #ifdef TIMING_OUTPUT ftime.stop(); cout << "Time to color" << ftime.totalElapsedTime() << endl; ftime.reset(); ftime.start(); #endif ssym->prober = prober; } #ifdef TIMING_OUTPUT symtime.stop(); cout << "Symbolic Time" << symtime.totalElapsedTime() << endl; symtime.reset(); #endif }
int checkmap(Epetra_Map & Map, int NumGlobalElements, int NumMyElements, int *MyGlobalElements, int IndexBase, Epetra_Comm& Comm, bool DistributedGlobal) { int i, ierr=0, forierr = 0; EPETRA_TEST_ERR(!Map.ConstantElementSize(),ierr); EPETRA_TEST_ERR(DistributedGlobal!=Map.DistributedGlobal(),ierr); EPETRA_TEST_ERR(Map.ElementSize()!=1,ierr); int *MyElementSizeList = new int[NumMyElements]; EPETRA_TEST_ERR(Map.ElementSizeList(MyElementSizeList)!=0,ierr); forierr = 0; for (i=0; i<NumMyElements; i++) forierr += MyElementSizeList[i]!=1; EPETRA_TEST_ERR(forierr,ierr); delete [] MyElementSizeList; const Epetra_Comm & Comm1 = Map.Comm(); EPETRA_TEST_ERR(Comm1.NumProc()!=Comm.NumProc(),ierr); EPETRA_TEST_ERR(Comm1.MyPID()!=Comm.MyPID(),ierr); EPETRA_TEST_ERR(Map.IndexBase()!=IndexBase,ierr); EPETRA_TEST_ERR(!Map.LinearMap() && MyGlobalElements==0,ierr); EPETRA_TEST_ERR(Map.LinearMap() && MyGlobalElements!=0,ierr); EPETRA_TEST_ERR(Map.MaxAllGID()!=NumGlobalElements-1+IndexBase,ierr); EPETRA_TEST_ERR(Map.MaxElementSize()!=1,ierr); int MaxLID = Map.MaxLID(); EPETRA_TEST_ERR(MaxLID!=NumMyElements-1,ierr); int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase; if (Comm.MyPID()>2) MaxMyGID+=3; if (!DistributedGlobal) MaxMyGID = NumMyElements-1+IndexBase; EPETRA_TEST_ERR(Map.MaxMyGID()!=MaxMyGID,ierr); EPETRA_TEST_ERR(Map.MinAllGID()!=IndexBase,ierr); EPETRA_TEST_ERR(Map.MinElementSize()!=1,ierr); EPETRA_TEST_ERR(Map.MinLID()!=0,ierr); int MinMyGID = Comm.MyPID()*NumMyElements+IndexBase; if (Comm.MyPID()>2) MinMyGID+=3; if (!DistributedGlobal) MinMyGID = 0; EPETRA_TEST_ERR(Map.MinMyGID()!=MinMyGID,ierr); int * MyGlobalElements1 = new int[NumMyElements]; EPETRA_TEST_ERR(Map.MyGlobalElements(MyGlobalElements1)!=0,ierr); forierr = 0; if (MyGlobalElements==0) { for (i=0; i<NumMyElements; i++) forierr += MyGlobalElements1[i]!=MinMyGID+i; EPETRA_TEST_ERR(forierr,ierr); } else { for (i=0; i<NumMyElements; i++) forierr += MyGlobalElements[i]!=MyGlobalElements1[i]; EPETRA_TEST_ERR(forierr,ierr); } EPETRA_TEST_ERR(Map.NumGlobalElements()!=NumGlobalElements,ierr); EPETRA_TEST_ERR(Map.NumGlobalPoints()!=NumGlobalElements,ierr); EPETRA_TEST_ERR(Map.NumMyElements()!=NumMyElements,ierr); EPETRA_TEST_ERR(Map.NumMyPoints()!=NumMyElements,ierr); int MaxMyGID2 = Map.GID(Map.LID(MaxMyGID)); EPETRA_TEST_ERR(MaxMyGID2 != MaxMyGID,ierr); int MaxLID2 = Map.LID(Map.GID(MaxLID)); EPETRA_TEST_ERR(MaxLID2 != MaxLID,ierr); EPETRA_TEST_ERR(Map.GID(MaxLID+1) != IndexBase-1,ierr);// MaxLID+1 doesn't exist EPETRA_TEST_ERR(Map.LID(MaxMyGID+1) != -1,ierr);// MaxMyGID+1 doesn't exist or is on a different processor EPETRA_TEST_ERR(!Map.MyGID(MaxMyGID),ierr); EPETRA_TEST_ERR(Map.MyGID(MaxMyGID+1),ierr); EPETRA_TEST_ERR(!Map.MyLID(MaxLID),ierr); EPETRA_TEST_ERR(Map.MyLID(MaxLID+1),ierr); EPETRA_TEST_ERR(!Map.MyGID(Map.GID(MaxLID)),ierr); EPETRA_TEST_ERR(Map.MyGID(Map.GID(MaxLID+1)),ierr); EPETRA_TEST_ERR(!Map.MyLID(Map.LID(MaxMyGID)),ierr); EPETRA_TEST_ERR(Map.MyLID(Map.LID(MaxMyGID+1)),ierr); // Check RemoteIDList function // Get some GIDs off of each processor to test int TotalNumEle, NumElePerProc, NumProc = Comm.NumProc(); int MinNumEleOnProc; int NumMyEle=Map.NumMyElements(); Comm.MinAll(&NumMyEle,&MinNumEleOnProc,1); if (MinNumEleOnProc > 5) NumElePerProc = 6; else NumElePerProc = MinNumEleOnProc; if (NumElePerProc > 0) { TotalNumEle = NumElePerProc*NumProc; int * MyGIDlist = new int[NumElePerProc]; int * GIDlist = new int[TotalNumEle]; int * PIDlist = new int[TotalNumEle]; int * LIDlist = new int[TotalNumEle]; for (i=0; i<NumElePerProc; i++) MyGIDlist[i] = MyGlobalElements1[i]; Comm.GatherAll(MyGIDlist,GIDlist,NumElePerProc);// Get a few values from each proc Map.RemoteIDList(TotalNumEle, GIDlist, PIDlist, LIDlist); int MyPID= Comm.MyPID(); forierr = 0; for (i=0; i<TotalNumEle; i++) { if (Map.MyGID(GIDlist[i])) { forierr += PIDlist[i] != MyPID; forierr += !Map.MyLID(Map.LID(GIDlist[i])) || Map.LID(GIDlist[i]) != LIDlist[i] || Map.GID(LIDlist[i]) != GIDlist[i]; } else { forierr += PIDlist[i] == MyPID; // If MyGID comes back false, the PID listed should be that of another proc } } EPETRA_TEST_ERR(forierr,ierr); delete [] MyGIDlist; delete [] GIDlist; delete [] PIDlist; delete [] LIDlist; } delete [] MyGlobalElements1; // Check RemoteIDList function (assumes all maps are linear, even if not stored that way) if (Map.LinearMap()) { int * GIDList = new int[3]; int * PIDList = new int[3]; int * LIDList = new int[3]; int MyPID = Map.Comm().MyPID(); int NumIDs = 0; //GIDList[NumIDs++] = Map.MaxAllGID()+1; // Should return -1 for both PID and LID if (Map.MinMyGID()-1>=Map.MinAllGID()) GIDList[NumIDs++] = Map.MinMyGID()-1; if (Map.MaxMyGID()+1<=Map.MaxAllGID()) GIDList[NumIDs++] = Map.MaxMyGID()+1; Map.RemoteIDList(NumIDs, GIDList, PIDList, LIDList); NumIDs = 0; //EPETRA_TEST_ERR(!(PIDList[NumIDs]==-1),ierr); //EPETRA_TEST_ERR(!(LIDList[NumIDs++]==-1),ierr); if (Map.MinMyGID()-1>=Map.MinAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs++]==MyPID-1),ierr); if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs]==MyPID+1),ierr); if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(LIDList[NumIDs++]==0),ierr); delete [] GIDList; delete [] PIDList; delete [] LIDList; } return (ierr); }
// ============================================================================ void EpetraExt::XMLWriter:: Write(const std::string& Label, const Epetra_Map& Map) { TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error, "No file has been opened"); long long NumGlobalElements = Map.NumGlobalElements64(); const int* MyGlobalElements_int = 0; const long long* MyGlobalElements_LL = 0; Map.MyGlobalElements(MyGlobalElements_int, MyGlobalElements_LL); if(!MyGlobalElements_int || !MyGlobalElements_LL) throw "EpetraExt::XMLWriter::Write: ERROR, GlobalIndices type unknown."; if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "<Map Label=\"" << Label << "\" NumElements=\"" << NumGlobalElements << '"' << " IndexBase=\"" << Map.IndexBase64() << '"' << " NumProc=\"" << Comm_.NumProc() << '"'; of.close(); } for (int iproc = 0; iproc < Comm_.NumProc(); ++iproc) { if (iproc == Comm_.MyPID()) { std::ofstream of(FileName_.c_str(), std::ios::app); of << " ElementsOnProc" << iproc << "=\"" << Map.NumMyElements() << '"'; of.close(); } Comm_.Barrier(); } if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << '>' << std::endl; of.close(); } for (int iproc = 0; iproc < Comm_.NumProc(); iproc++) { if (iproc == Comm_.MyPID()) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "<Proc ID=\"" << Comm_.MyPID() << "\">" << std::endl; if(MyGlobalElements_int) { for (int i = 0; i < Map.NumMyElements(); ++i) { of << MyGlobalElements_int[i] << std::endl; } } else { for (int i = 0; i < Map.NumMyElements(); ++i) { of << MyGlobalElements_LL[i] << std::endl; } } of << "</Proc>" << std::endl; of.close(); } Comm_.Barrier(); } if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "</Map>" << std::endl; of.close(); } }
int main(int argc, char *argv[]) { using Teuchos::RCP; // reference count pointers using Teuchos::rcp; // // // MPI initialization using Teuchos // #ifdef HAVE_MPI MPI_Init(&argc, &argv); Epetra_MpiComm comm(MPI_COMM_WORLD); #else Epetra_SerialComm comm; #endif // // Parameters // GlobalOrdinal numGlobalElements = 256; // problem size // // Construct the problem // // Construct a Map that puts approximately the same number of equations on each processor const Epetra_Map map(numGlobalElements, 0, comm); // Get update list and number of local equations from newly created map. const size_t numMyElements = map.NumMyElements(); const GlobalOrdinal* myGlobalElements = map.MyGlobalElements(); // Create a CrsMatrix using the map, with a dynamic allocation of 3 entries per row RCP<Epetra_CrsMatrix> A = rcp(new Epetra_CrsMatrix(Copy, map, 3)); // Add rows one-at-a-time for (size_t i = 0; i < numMyElements; i++) { if (myGlobalElements[i] == 0) { //TODO: should be rewritten in an Epetra style A->InsertGlobalValues(myGlobalElements[i], 2, Teuchos::tuple<Scalar> (2.0, -1.0).getRawPtr(), Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i], myGlobalElements[i] +1).getRawPtr()); } else if (myGlobalElements[i] == numGlobalElements - 1) { A->InsertGlobalValues(myGlobalElements[i], 2, Teuchos::tuple<Scalar> (-1.0, 2.0).getRawPtr(), Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i] -1, myGlobalElements[i]).getRawPtr()); } else { A->InsertGlobalValues(myGlobalElements[i], 3, Teuchos::tuple<Scalar> (-1.0, 2.0, -1.0).getRawPtr(), Teuchos::tuple<GlobalOrdinal>(myGlobalElements[i] -1, myGlobalElements[i], myGlobalElements[i] +1).getRawPtr()); } } // Complete the fill, ask that storage be reallocated and optimized A->FillComplete(); // // Construct a multigrid preconditioner // // Turns a Epetra_CrsMatrix into a MueLu::Matrix RCP<Xpetra::CrsMatrix<SC, LO, GO, NO, LMO> > mueluA_ = rcp(new Xpetra::EpetraCrsMatrix(A)); //TODO: should not be needed RCP<Xpetra::Matrix <SC, LO, GO, NO, LMO> > mueluA = rcp(new Xpetra::CrsMatrixWrap<SC, LO, GO, NO, LMO>(mueluA_)); // Multigrid Hierarchy RCP<Hierarchy> H = rcp(new Hierarchy(mueluA)); H->setVerbLevel(Teuchos::VERB_HIGH); // Multigrid setup phase (using default parameters) H->Setup(); // // Define RHS / LHS // RCP<Epetra_Vector> X = rcp(new Epetra_Vector(map)); RCP<Epetra_Vector> B = rcp(new Epetra_Vector(map)); X->PutScalar((Scalar) 0.0); B->SetSeed(846930886); B->Random(); #ifndef HAVE_MUELU_BELOS // // Use AMG directly as an iterative solver (not as a preconditionner) // int nIts = 9; // Wrap Epetra Vectors into Xpetra Vectors RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(X)); RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(B)); H->Iterate(*mueluB, nIts, *mueluX); // Print relative residual norm ST::magnitudeType residualNorms = Utils::ResidualNorm(*mueluA, *mueluX, *mueluB)[0]; if (comm.MyPID() == 0) std::cout << "||Residual|| = " << residualNorms << std::endl; #else // HAVE_MUELU_BELOS // // Solve Ax = b using AMG as a preconditioner in Belos // // Matrix and Multivector type that will be used with Belos typedef Epetra_MultiVector MV; typedef Belos::OperatorT<MV> OP; // Define Operator and Preconditioner RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(mueluA)); // Turns a Xpetra::Matrix object into a Belos operator RCP<OP> belosPrec = rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator // Construct a Belos LinearProblem object RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B)); belosProblem->setLeftPrec(belosPrec); bool set = belosProblem->setProblem(); if (set == false) { std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl; return EXIT_FAILURE; } // Belos parameter list int maxIts = 20; double tol = 1e-4; Teuchos::ParameterList belosList; belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails); // Create an iterative solver manager RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false))); // Perform solve Belos::ReturnType ret = solver->solve(); // Get the number of iterations for this solve. std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl; // Compute actual residuals. int numrhs=1; bool badRes = false; std::vector<SC> actual_resids(numrhs); std::vector<SC> rhs_norm(numrhs); RCP<Epetra_MultiVector > resid = rcp(new Epetra_MultiVector(map, numrhs)); typedef Belos::OperatorTraits<SC, MV, OP> OPT; typedef Belos::MultiVecTraits<SC, MV> MVT; OPT::Apply(*belosOp, *X, *resid); MVT::MvAddMv(-1.0, *resid, 1.0, *B, *resid); MVT::MvNorm(*resid, actual_resids); MVT::MvNorm(*B, rhs_norm); std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl; for (int i = 0; i < numrhs; i++) { SC actRes = actual_resids[i]/rhs_norm[i]; std::cout <<"Problem " << i << " : \t" << actRes <<std::endl; if (actRes > tol) { badRes = true; } } // Check convergence if (ret != Belos::Converged || badRes) { std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl; return EXIT_FAILURE; } std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl; #endif // HAVE_MUELU_BELOS #ifdef HAVE_MPI MPI_Finalize(); #endif return EXIT_SUCCESS; }
int main (int argc, char *argv[]) { // These "using" statements make the code a bit more concise. using std::cout; using std::endl; int ierr = 0, i; // If Trilinos was built with MPI, initialize MPI, otherwise // initialize the serial "communicator" that stands in for MPI. #ifdef EPETRA_MPI MPI_Init (&argc,&argv); Epetra_MpiComm Comm (MPI_COMM_WORLD); #else Epetra_SerialComm Comm; #endif const int MyPID = Comm.MyPID(); const int NumProc = Comm.NumProc(); // We only allow (MPI) Process 0 to write to stdout. const bool verbose = (MyPID == 0); const int NumGlobalElements = 100; if (verbose) cout << Epetra_Version() << endl << endl; // Asking the Epetra_Comm to print itself is a good test for whether // you are running in an MPI environment. However, it will print // something on all MPI processes, so you should remove it for a // large-scale parallel run. cout << Comm << endl; if (NumGlobalElements < NumProc) { if (verbose) cout << "numGlobalBlocks = " << NumGlobalElements << " cannot be < number of processors = " << NumProc << endl; std::exit (EXIT_FAILURE); } // Construct a Map that puts approximately the same number of rows // of the matrix A on each processor. Epetra_Map Map (NumGlobalElements, 0, Comm); // Get update list and number of local equations from newly created Map. int NumMyElements = Map.NumMyElements(); std::vector<int> MyGlobalElements(NumMyElements); Map.MyGlobalElements(&MyGlobalElements[0]); // NumNz[i] is the number of nonzero elements in row i of the sparse // matrix on this MPI process. Epetra_CrsMatrix uses this to figure // out how much space to allocate. std::vector<int> NumNz (NumMyElements); // We are building a tridiagonal matrix where each row contains the // nonzero elements (-1 2 -1). Thus, we need 2 off-diagonal terms, // except for the first and last row of the matrix. for (int i = 0; i < NumMyElements; ++i) if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1) NumNz[i] = 2; // First or last row else NumNz[i] = 3; // Not the (first or last row) // Create the Epetra_CrsMatrix. Epetra_CrsMatrix A (Copy, Map, &NumNz[0]); // // Add rows to the sparse matrix one at a time. // std::vector<double> Values(2); Values[0] = -1.0; Values[1] = -1.0; std::vector<int> Indices(2); const double two = 2.0; int NumEntries; for (int i = 0; i < NumMyElements; ++i) { if (MyGlobalElements[i] == 0) { // The first row of the matrix. Indices[0] = 1; NumEntries = 1; } else if (MyGlobalElements[i] == NumGlobalElements - 1) { // The last row of the matrix. Indices[0] = NumGlobalElements-2; NumEntries = 1; } else { // Any row of the matrix other than the first or last. Indices[0] = MyGlobalElements[i]-1; Indices[1] = MyGlobalElements[i]+1; NumEntries = 2; } ierr = A.InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); assert (ierr==0); // Insert the diagonal entry. ierr = A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i]); assert(ierr==0); } // Finish up. We can call FillComplete() with no arguments, because // the matrix is square. ierr = A.FillComplete (); assert (ierr==0); // Parameters for the power method. const int niters = NumGlobalElements*10; const double tolerance = 1.0e-2; // // Run the power method. Keep track of the flop count and the total // elapsed time. // Epetra_Flops counter; A.SetFlopCounter(counter); Epetra_Time timer(Comm); double lambda = 0.0; ierr += powerMethod (lambda, A, niters, tolerance, verbose); double elapsedTime = timer.ElapsedTime (); double totalFlops =counter.Flops (); // Mflop/s: Million floating-point arithmetic operations per second. double Mflop_per_s = totalFlops / elapsedTime / 1000000.0; if (verbose) cout << endl << endl << "Total Mflop/s for first solve = " << Mflop_per_s << endl<< endl; // Increase the first (0,0) diagonal entry of the matrix. if (verbose) cout << endl << "Increasing magnitude of first diagonal term, solving again" << endl << endl << endl; if (A.MyGlobalRow (0)) { int numvals = A.NumGlobalEntries (0); std::vector<double> Rowvals (numvals); std::vector<int> Rowinds (numvals); A.ExtractGlobalRowCopy (0, numvals, numvals, &Rowvals[0], &Rowinds[0]); // Get A(0,0) for (int i = 0; i < numvals; ++i) if (Rowinds[i] == 0) Rowvals[i] *= 10.0; A.ReplaceGlobalValues (0, numvals, &Rowvals[0], &Rowinds[0]); } // // Run the power method again. Keep track of the flop count and the // total elapsed time. // lambda = 0.0; timer.ResetStartTime(); counter.ResetFlops(); ierr += powerMethod (lambda, A, niters, tolerance, verbose); elapsedTime = timer.ElapsedTime(); totalFlops = counter.Flops(); Mflop_per_s = totalFlops / elapsedTime / 1000000.0; if (verbose) cout << endl << endl << "Total Mflop/s for second solve = " << Mflop_per_s << endl << endl; #ifdef EPETRA_MPI MPI_Finalize() ; #endif return ierr; }
// // Diagonal: 0=no change, 1=eliminate entry // from the map for the largest row element in process 0 // 2=add diagonal entries to the matrix, with a zero value // (assume row map contains all diagonal entries). // // ReindexRowMap: // 0=no change, 1= add 2 (still contiguous), 2=non-contiguous // // ReindexColMap // 0=same as RowMap, 1=add 4 - Different From RowMap, but contiguous) // // RangeMap: // 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map // // DomainMap: // 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map // RCP<Epetra_CrsMatrix> NewMatNewMap(Epetra_CrsMatrix& In, int Diagonal, int ReindexRowMap, int ReindexColMap, int RangeMapType, int DomainMapType ) { // // If we are making no change, return the original matrix (which has a linear map) // #if 0 std::cout << __FILE__ << "::" << __LINE__ << " " << Diagonal << " " << ReindexRowMap << " " << ReindexColMap << " " << RangeMapType << " " << DomainMapType << " " << std::endl ; #endif if ( Diagonal + ReindexRowMap + ReindexColMap + RangeMapType + DomainMapType == 0 ) { RCP<Epetra_CrsMatrix> ReturnOrig = rcp( &In, false ); return ReturnOrig ; } // // Diagonal==2 is used for a different purpose - // Making sure that the diagonal of the matrix is non-empty. // Note: The diagonal must exist in In.RowMap(). // if ( Diagonal == 2 ) { assert( ReindexRowMap==0 && ReindexColMap == 0 ) ; } int (*RowPermute)(int in) = 0; int (*ColPermute)(int in) = 0; assert( Diagonal >= 0 && Diagonal <= 2 ); assert( ReindexRowMap>=0 && ReindexRowMap<=2 ); assert( ReindexColMap>=0 && ReindexColMap<=1 ); assert( RangeMapType>=0 && RangeMapType<=3 ); assert( DomainMapType>=0 && DomainMapType<=3 ); Epetra_Map DomainMap = In.DomainMap(); Epetra_Map RangeMap = In.RangeMap(); Epetra_Map ColMap = In.ColMap(); Epetra_Map RowMap = In.RowMap(); int NumMyRowElements = RowMap.NumMyElements(); int NumMyColElements = ColMap.NumMyElements(); int NumMyRangeElements = RangeMap.NumMyElements(); int NumMyDomainElements = DomainMap.NumMyElements(); int NumGlobalRowElements = RowMap.NumGlobalElements(); int NumGlobalColElements = ColMap.NumGlobalElements(); int NumGlobalRangeElements = RangeMap.NumGlobalElements(); int NumGlobalDomainElements = DomainMap.NumGlobalElements(); assert( NumGlobalRangeElements == NumGlobalDomainElements ) ; std::vector<int> MyGlobalRowElements( NumMyRowElements ) ; std::vector<int> NumEntriesPerRow( NumMyRowElements ) ; std::vector<int> MyPermutedGlobalRowElements( NumMyRowElements ) ; std::vector<int> MyGlobalColElements( NumMyColElements ) ; std::vector<int> MyPermutedGlobalColElements( NumMyColElements ) ; // Used to create the column map std::vector<int> MyPermutedGlobalColElementTable( NumMyColElements ) ; // To convert local indices to global std::vector<int> MyGlobalRangeElements( NumMyRangeElements ) ; std::vector<int> MyPermutedGlobalRangeElements( NumMyRangeElements ) ; std::vector<int> MyGlobalDomainElements( NumMyDomainElements ) ; std::vector<int> MyPermutedGlobalDomainElements( NumMyDomainElements ) ; RowMap.MyGlobalElements(&MyGlobalRowElements[0]); ColMap.MyGlobalElements(&MyGlobalColElements[0]); RangeMap.MyGlobalElements(&MyGlobalRangeElements[0]); DomainMap.MyGlobalElements(&MyGlobalDomainElements[0]); switch( ReindexRowMap ) { case 0: RowPermute = &NoPermute ; break; case 1: RowPermute = &SmallRowPermute ; break; case 2: RowPermute = BigRowPermute ; break; } switch( ReindexColMap ) { case 0: ColPermute = RowPermute ; break; case 1: ColPermute = &SmallColPermute ; break; } // // Create Serial Range and Domain Maps based on the permuted indexing // int nlocal = 0; if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements; std::vector<int> AllIDs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDs[i] = (*RowPermute)( i ) ; Epetra_Map SerialRangeMap( -1, nlocal, &AllIDs[0], 0, In.Comm()); std::vector<int> AllIDBs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDBs[i] = (*ColPermute)( i ) ; Epetra_Map SerialDomainMap( -1, nlocal, &AllIDBs[0], 0, In.Comm()); // // Create Bizarre Range and Domain Maps based on the permuted indexing // These are nearly serial, having all but one element on process 0 // The goal here is to make sure that we can use Domain and Range maps // that are neither serial, nor distributed in the normal manner. // std::vector<int> AllIDCs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDCs[i] = (*ColPermute)( i ) ; if ( In.Comm().NumProc() > 1 ) { if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1; if (In.Comm().MyPID()==1) { nlocal = 1; AllIDCs[0] = (*ColPermute)( NumGlobalRangeElements - 1 ); } } int iam = In.Comm().MyPID(); Epetra_Map BizarreDomainMap( -1, nlocal, &AllIDCs[0], 0, In.Comm()); std::vector<int> AllIDDs( NumGlobalRangeElements ) ; for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDDs[i] = (*RowPermute)( i ) ; if ( In.Comm().NumProc() > 1 ) { if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1; if (In.Comm().MyPID()==1) { nlocal = 1; AllIDDs[0] = (*RowPermute)( NumGlobalRangeElements -1 ) ; } } Epetra_Map BizarreRangeMap( -1, nlocal, &AllIDDs[0], 0, In.Comm()); // // Compute the column map // // If Diagonal==1, remove the column corresponding to the last row owned // by process 0. Removing this column from a tridiagonal matrix, leaves // a disconnected, but non-singular matrix. // int NumMyColElementsOut = 0 ; int NumGlobalColElementsOut ; if ( Diagonal == 1 ) NumGlobalColElementsOut = NumGlobalColElements-1; else NumGlobalColElementsOut = NumGlobalColElements; if ( Diagonal == 1 && iam==0 ) { for ( int i=0; i < NumMyColElements ; i++ ) { if ( MyGlobalColElements[i] != MyGlobalRowElements[NumMyRowElements-1] ) { MyPermutedGlobalColElements[NumMyColElementsOut++] = (*ColPermute)( MyGlobalColElements[i] ) ; } } assert( NumMyColElementsOut == NumMyColElements-1 ); } else { for ( int i=0; i < NumMyColElements ; i++ ) MyPermutedGlobalColElements[i] = (*ColPermute)( MyGlobalColElements[i] ) ; NumMyColElementsOut = NumMyColElements ; if ( Diagonal == 2 ) { // For each row, make sure that the column map has this row in it, // if it doesn't, add it to the column map. // Note: MyPermutedGlobalColElements == MyGlobalColElements when // Diagonal==2 because ( Diagonal == 2 ) implies: // ReindexRowMap==0 && ReindexColMap == 0 - see assert above for ( int i=0; i < NumMyRowElements ; i++ ) { bool MissingDiagonal = true; for ( int j=0; j < NumMyColElements; j++ ) { if ( MyGlobalRowElements[i] == MyGlobalColElements[j] ) { MissingDiagonal = false; } } if ( MissingDiagonal ) { MyPermutedGlobalColElements.resize(NumMyColElements+1); MyPermutedGlobalColElements[NumMyColElementsOut] = MyGlobalRowElements[i]; NumMyColElementsOut++; } } In.Comm().SumAll(&NumMyColElementsOut,&NumGlobalColElementsOut,1); } } // // These tables are used both as the permutation tables and to create the maps. // for ( int i=0; i < NumMyColElements ; i++ ) MyPermutedGlobalColElementTable[i] = (*ColPermute)( MyGlobalColElements[i] ) ; for ( int i=0; i < NumMyRowElements ; i++ ) MyPermutedGlobalRowElements[i] = (*RowPermute)( MyGlobalRowElements[i] ) ; for ( int i=0; i < NumMyRangeElements ; i++ ) MyPermutedGlobalRangeElements[i] = (*RowPermute)( MyGlobalRangeElements[i] ) ; for ( int i=0; i < NumMyDomainElements ; i++ ) MyPermutedGlobalDomainElements[i] = (*ColPermute)( MyGlobalDomainElements[i] ) ; RCP<Epetra_Map> PermutedRowMap = rcp( new Epetra_Map( NumGlobalRowElements, NumMyRowElements, &MyPermutedGlobalRowElements[0], 0, In.Comm() ) ); RCP<Epetra_Map> PermutedColMap = rcp( new Epetra_Map( NumGlobalColElementsOut, NumMyColElementsOut, &MyPermutedGlobalColElements[0], 0, In.Comm() ) ); RCP<Epetra_Map> PermutedRangeMap = rcp( new Epetra_Map( NumGlobalRangeElements, NumMyRangeElements, &MyPermutedGlobalRangeElements[0], 0, In.Comm() ) ); RCP<Epetra_Map> PermutedDomainMap = rcp( new Epetra_Map( NumGlobalDomainElements, NumMyDomainElements, &MyPermutedGlobalDomainElements[0], 0, In.Comm() ) ); // // These vectors are filled and then passed to InsertGlobalValues // std::vector<int> ThisRowIndices( In.MaxNumEntries() ); std::vector<double> ThisRowValues( In.MaxNumEntries() ); std::vector<int> PermutedGlobalColIndices( In.MaxNumEntries() ); //std::cout << __FILE__ << "::" <<__LINE__ << std::endl ; RCP<Epetra_CrsMatrix> Out = rcp( new Epetra_CrsMatrix( Copy, *PermutedRowMap, *PermutedColMap, 0 ) ); for (int i=0; i<NumMyRowElements; i++) { int NumIndicesThisRow = 0; assert( In.ExtractMyRowCopy( i, In.MaxNumEntries(), NumIndicesThisRow, &ThisRowValues[0], &ThisRowIndices[0] ) == 0 ) ; for (int j = 0 ; j < NumIndicesThisRow ; j++ ) { PermutedGlobalColIndices[j] = MyPermutedGlobalColElementTable[ ThisRowIndices[j] ] ; } bool MissingDiagonal = false; if ( Diagonal==2 ) { // assert( MyGlobalRowElements[i] == MyPermutedGlobalRowElements[i] ); MissingDiagonal = true; for( int j =0 ; j < NumIndicesThisRow ; j++ ) { if ( PermutedGlobalColIndices[j] == MyPermutedGlobalRowElements[i] ) { MissingDiagonal = false ; } } #if 0 std::cout << __FILE__ << "::" << __LINE__ << " i = " << i << " MyPermutedGlobalRowElements[i] = " << MyPermutedGlobalRowElements[i] << " MissingDiagonal = " << MissingDiagonal << std::endl ; #endif } if ( MissingDiagonal ) { ThisRowValues.resize(NumIndicesThisRow+1) ; ThisRowValues[NumIndicesThisRow] = 0.0; PermutedGlobalColIndices.resize(NumIndicesThisRow+1); PermutedGlobalColIndices[NumIndicesThisRow] = MyPermutedGlobalRowElements[i] ; #if 0 std::cout << __FILE__ << "::" << __LINE__ << " i = " << i << "NumIndicesThisRow = " << NumIndicesThisRow << "ThisRowValues[NumIndicesThisRow = " << ThisRowValues[NumIndicesThisRow] << " PermutedGlobalColIndices[NumIndcesThisRow] = " << PermutedGlobalColIndices[NumIndicesThisRow] << std::endl ; #endif NumIndicesThisRow++ ; } assert( Out->InsertGlobalValues( MyPermutedGlobalRowElements[i], NumIndicesThisRow, &ThisRowValues[0], &PermutedGlobalColIndices[0] ) >= 0 ); } // Epetra_LocalMap ReplicatedMap( NumGlobalRangeElements, 0, In.Comm() ); RCP<Epetra_Map> OutRangeMap ; RCP<Epetra_Map> OutDomainMap ; switch( RangeMapType ) { case 0: OutRangeMap = PermutedRangeMap ; break; case 1: OutRangeMap = rcp(&SerialRangeMap, false); break; case 2: OutRangeMap = rcp(&BizarreRangeMap, false); break; case 3: OutRangeMap = rcp(&ReplicatedMap, false); break; } // switch( DomainMapType ) { switch( DomainMapType ) { case 0: OutDomainMap = PermutedDomainMap ; break; case 1: OutDomainMap = rcp(&SerialDomainMap, false); break; case 2: OutDomainMap = rcp(&BizarreDomainMap, false); break; case 3: OutDomainMap = rcp(&ReplicatedMap, false); break; } #if 0 assert(Out->FillComplete( *PermutedDomainMap, *PermutedRangeMap )==0); #else assert(Out->FillComplete( *OutDomainMap, *OutRangeMap )==0); #endif #if 0 std::cout << __FILE__ << "::" << __LINE__ << std::endl ; Out->Print( std::cout ) ; #endif return Out; }
Epetra_CrsMatrix * Laplace2D::CreateLaplacian( const int nx, const int ny, const Epetra_Comm * Comm) { int NumGlobalElements = nx * ny; // create a map Epetra_Map * Map = new Epetra_Map(NumGlobalElements,0,*Comm); // local number of rows int NumMyElements = Map->NumMyElements(); // get update list int * MyGlobalElements = Map->MyGlobalElements(); double hx = 1.0/(nx-1); double hy = 1.0/(ny-1); double off_left = -1.0/(hx*hx); double off_right = -1.0/(hx*hx); double off_lower = -1.0/(hy*hy); double off_upper = -1.0/(hy*hy); double diag = 2.0/(hx*hx) + 2.0/(hy*hy); int left, right, lower, upper; // a bit overestimated the nonzero per row Epetra_CrsMatrix * A = new Epetra_CrsMatrix(Copy,*Map,5); // Add rows one-at-a-time double * Values = new double[4]; int * Indices = new int[4]; for( int i = 0; i < NumMyElements; ++i ) { int NumEntries=0; get_myNeighbours( MyGlobalElements[i], nx, ny, left, right, lower, upper ); if( left != -1 ) { Indices[NumEntries] = left; Values[NumEntries] = off_left; ++NumEntries; } if( right != -1 ) { Indices[NumEntries] = right; Values[NumEntries] = off_right; ++NumEntries; } if( lower != -1 ) { Indices[NumEntries] = lower; Values[NumEntries] = off_lower; ++NumEntries; } if( upper != -1 ) { Indices[NumEntries] = upper; Values[NumEntries] = off_upper; ++NumEntries; } // put the off-diagonal entries A->InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices); // Put in the diagonal entry A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, MyGlobalElements+i); } // put matrix in local ordering A->FillComplete(); delete [] Indices; delete [] Values; delete Map; return A; } /* createJacobian */
int TLowCommunicationMakeColMapAndReindex(int N, const int * rowptr, int * colind_LID, const int_type *colind_GID, const Epetra_Map& domainMap, const int * owningPIDs, bool SortGhostsAssociatedWithEachProcessor, std::vector<int>& RemotePIDs, MapType1 & NewColMap) { int i,j; // Sanity checks bool UseLL; if(domainMap.GlobalIndicesLongLong()) UseLL=true; else if(domainMap.GlobalIndicesInt()) UseLL=false; else throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot detect int type."); // Scan all column indices and sort into two groups: // Local: those whose GID matches a GID of the domain map on this processor and // Remote: All others. int numDomainElements = domainMap.NumMyElements(); bool * LocalGIDs = 0; if (numDomainElements>0) LocalGIDs = new bool[numDomainElements]; for (i=0; i<numDomainElements; i++) LocalGIDs[i] = false; // Assume domain GIDs are not local bool DoSizes = !domainMap.ConstantElementSize(); // If not constant element size, then error if(DoSizes) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot handle non-constant sized domainMap."); // In principle it is good to have RemoteGIDs and RemotGIDList be as long as the number of remote GIDs // on this processor, but this would require two passes through the column IDs, so we make it the max of 100 // and the number of block rows. const int numMyBlockRows = N; int hashsize = numMyBlockRows; if (hashsize < 100) hashsize = 100; Epetra_HashTable<int_type> RemoteGIDs(hashsize); std::vector<int_type> RemoteGIDList; RemoteGIDList.reserve(hashsize); std::vector<int> PIDList; PIDList.reserve(hashsize); // Here we start using the *int* colind array. If int_type==int this clobbers the GIDs, if // int_type==long long, then this is the first use of the colind array. // For *local* GID's set colind with with their LID in the domainMap. For *remote* GIDs, // we set colind with (numDomainElements+NumRemoteColGIDs) before the increment of // the remote count. These numberings will be separate because no local LID is greater // than numDomainElements. int NumLocalColGIDs = 0; int NumRemoteColGIDs = 0; for(i = 0; i < numMyBlockRows; i++) { for(j = rowptr[i]; j < rowptr[i+1]; j++) { int_type GID = colind_GID[j]; // Check if GID matches a row GID int LID = domainMap.LID(GID); if(LID != -1) { bool alreadyFound = LocalGIDs[LID]; if (!alreadyFound) { LocalGIDs[LID] = true; // There is a column in the graph associated with this domain map GID NumLocalColGIDs++; } colind_LID[j] = LID; } else { int_type hash_value=RemoteGIDs.Get(GID); if(hash_value == -1) { // This means its a new remote GID int PID = owningPIDs[j]; if(PID==-1) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: Cannot figure out if PID is owned."); colind_LID[j] = numDomainElements + NumRemoteColGIDs; RemoteGIDs.Add(GID, NumRemoteColGIDs); RemoteGIDList.push_back(GID); PIDList.push_back(PID); NumRemoteColGIDs++; } else colind_LID[j] = numDomainElements + hash_value; } } } // Possible short-circuit: If all domain map GIDs are present as column indices, then set ColMap=domainMap and quit if (domainMap.Comm().NumProc()==1) { if (NumRemoteColGIDs!=0) { throw std::runtime_error("Some column IDs are not in domainMap. If matrix is rectangular, you must pass in a domainMap"); // Sanity test: When one processor,there can be no remoteGIDs } if (NumLocalColGIDs==numDomainElements) { if (LocalGIDs!=0) delete [] LocalGIDs; // In this case, we just use the domainMap's indices, which is, not coincidently, what we clobbered colind with up above anyway. // No further reindexing is needed. NewColMap = domainMap; return 0; } } // Now build integer array containing column GIDs // Build back end, containing remote GIDs, first int numMyBlockCols = NumLocalColGIDs + NumRemoteColGIDs; std::vector<int_type> ColIndices; int_type * RemoteColIndices=0; if(numMyBlockCols > 0) { ColIndices.resize(numMyBlockCols); if(NumLocalColGIDs!=numMyBlockCols) RemoteColIndices = &ColIndices[NumLocalColGIDs]; // Points to back end of ColIndices else RemoteColIndices=0; } for(i = 0; i < NumRemoteColGIDs; i++) RemoteColIndices[i] = RemoteGIDList[i]; // Build permute array for *remote* reindexing. std::vector<int> RemotePermuteIDs(NumRemoteColGIDs); for(i=0; i<NumRemoteColGIDs; i++) RemotePermuteIDs[i]=i; // Sort External column indices so that all columns coming from a given remote processor are contiguous int NumListsInt=0; int NumListsLL =0; int * IntSortLists[2]; long long * LLSortLists[2]; int * RemotePermuteIDs_ptr = RemotePermuteIDs.size() ? &RemotePermuteIDs[0] : 0; if(!UseLL) { // int version IntSortLists[0] = (int*) RemoteColIndices; IntSortLists[1] = RemotePermuteIDs_ptr; NumListsInt=2; } else { //LL version LLSortLists[0] = (long long*) RemoteColIndices; IntSortLists[0] = RemotePermuteIDs_ptr; NumListsInt = NumListsLL = 1; } int * PIDList_ptr = PIDList.size() ? &PIDList[0] : 0; Epetra_Util::Sort(true, NumRemoteColGIDs, PIDList_ptr, 0, 0, NumListsInt, IntSortLists,NumListsLL,LLSortLists); // Stash the RemotePIDs PIDList.resize(NumRemoteColGIDs); RemotePIDs = PIDList; if (SortGhostsAssociatedWithEachProcessor) { // Sort external column indices so that columns from a given remote processor are not only contiguous // but also in ascending order. NOTE: I don't know if the number of externals associated // with a given remote processor is known at this point ... so I count them here. // NTS: Only sort the RemoteColIndices this time... int StartCurrent, StartNext; StartCurrent = 0; StartNext = 1; while ( StartNext < NumRemoteColGIDs ) { if (PIDList[StartNext]==PIDList[StartNext-1]) StartNext++; else { IntSortLists[0] = &RemotePermuteIDs[StartCurrent]; Epetra_Util::Sort(true,StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]),0,0,1,IntSortLists,0,0); StartCurrent = StartNext; StartNext++; } } IntSortLists[0] = &RemotePermuteIDs[StartCurrent]; Epetra_Util::Sort(true, StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]), 0, 0, 1,IntSortLists,0,0); } // Reverse the permutation to get the information we actually care about std::vector<int> ReverseRemotePermuteIDs(NumRemoteColGIDs); for(i=0; i<NumRemoteColGIDs; i++) ReverseRemotePermuteIDs[RemotePermuteIDs[i]]=i; // Build permute array for *local* reindexing. bool use_local_permute=false; std::vector<int> LocalPermuteIDs(numDomainElements); // Now fill front end. Two cases: // (1) If the number of Local column GIDs is the same as the number of Local domain GIDs, we // can simply read the domain GIDs into the front part of ColIndices, otherwise // (2) We step through the GIDs of the domainMap, checking to see if each domain GID is a column GID. // we want to do this to maintain a consistent ordering of GIDs between the columns and the domain. if(NumLocalColGIDs == domainMap.NumMyElements()) { if(NumLocalColGIDs > 0) { domainMap.MyGlobalElements(&ColIndices[0]); // Load Global Indices into first numMyBlockCols elements column GID list } } else { int_type* MyGlobalElements = 0; domainMap.MyGlobalElementsPtr(MyGlobalElements); int* ElementSizeList = 0; if(DoSizes) ElementSizeList = domainMap.ElementSizeList(); int NumLocalAgain = 0; use_local_permute = true; for(i = 0; i < numDomainElements; i++) { if(LocalGIDs[i]) { LocalPermuteIDs[i] = NumLocalAgain; ColIndices[NumLocalAgain++] = MyGlobalElements[i]; } } assert(NumLocalAgain==NumLocalColGIDs); // Sanity test } // Done with this array if (LocalGIDs!=0) delete [] LocalGIDs; // Make Column map with same element sizes as Domain map int_type * ColIndices_ptr = ColIndices.size() ? &ColIndices[0] : 0; MapType2 temp((int_type)(-1), numMyBlockCols, ColIndices_ptr, (int_type)domainMap.IndexBase64(), domainMap.Comm()); NewColMap = temp; // Low-cost reindex of the matrix for(i=0; i<numMyBlockRows; i++){ for(j=rowptr[i]; j<rowptr[i+1]; j++){ int ID=colind_LID[j]; if(ID < numDomainElements){ if(use_local_permute) colind_LID[j] = LocalPermuteIDs[colind_LID[j]]; // In the case where use_local_permute==false, we just copy the DomainMap's ordering, which it so happens // is what we put in colind to begin with. } else colind_LID[j] = NumLocalColGIDs + ReverseRemotePermuteIDs[colind_LID[j]-numDomainElements]; } } return 0; }
int main(int argc, char *argv[]) { #ifdef HAVE_MPI MPI_Init(&argc,&argv); Epetra_MpiComm Comm( MPI_COMM_WORLD ); #else Epetra_SerialComm Comm; #endif // My MPI process rank. const int MyPID = Comm.MyPID(); // "/Users/sakashitatatsuya/Downloads/barista_trunk_slepc/sample/hamiltonian_matrix.ip" std::ifstream ifs(argv[1]); alps::Parameters params(ifs); Teuchos::oblackholestream blackHole; std::ostream& out = (MyPID == 0) ? std::cout : blackHole; barista::Hamiltonian<> hamiltonian(params); matrix_type matrix(hamiltonian.dimension(), hamiltonian.dimension()); hamiltonian.fill<double>(matrix); int m,n; int N; m = n = N = hamiltonian.dimension(); //std::cout << matrix << std::endl; std::ofstream ofs; if (MyPID==0) { ofs.open("anasazi_time.txt"); if (!ofs) { #ifdef HAVE_MPI MPI_Finalize() ; #endif return -1; } } //Teuchos::ParameterList GaleriList; using Teuchos::RCP; using Teuchos::rcp; typedef Teuchos::ScalarTraits<double> STS; const double one = STS::one(); const double zero = STS::zero(); // The problem is defined on a 2D grid, global size is nx * nx. //int nx = N; //GaleriList.set("n", nx * nx); //GaleriList.set("nx", nx); //GaleriList.set("ny", nx); //Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( Galeri::CreateMap("Linear", Comm, GaleriList) ); //Teuchos::RCP<Epetra_RowMatrix> A = Teuchos::rcp( Galeri::CreateCrsMatrix("Laplace2D", &*Map, GaleriList) ); // Construct a Map that puts approximately the same number of rows // of the matrix A on each processor. Epetra_Map RowMap (N, 0, Comm); Epetra_Map ColMap (N, 0, Comm); // Get update list and number of local equations from newly created Map. const int NumMyRowElements = RowMap.NumMyElements (); std::vector<int> MyGlobalRowElements (NumMyRowElements); RowMap.MyGlobalElements (&MyGlobalRowElements[0]); // Create an Epetra_CrsMatrix using the given row map. RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Copy, RowMap, n)); // We use info to catch any errors that may have happened during // matrix assembly, and report them globally. We do this so that // the MPI processes won't call FillComplete() unless they all // successfully filled their parts of the matrix. int info = 0; try { // // Compute coefficients for the discrete integral operator. // std::vector<double> Values (n); std::vector<int> Indices (n); //const double inv_mp1 = one / (m+1); //const double inv_np1 = one / (n+1); int count; //for (int i = 0; i < n; ++i) { // Indices[i] = i; //} for (int i = 0; i < NumMyRowElements; ++i) { count =0; for (int j = 0; j < n; ++j) { if (matrix(MyGlobalRowElements[i],j)!=0) { Values[count] = matrix(MyGlobalRowElements[i],j); Indices[count] = j; count++; } } info = A->InsertGlobalValues (MyGlobalRowElements[i], count, &Values[0], &Indices[0]); // Make sure that the insertion succeeded. Teuchos' // TEST_FOR_EXCEPTION macro gives a nice error message if the // thrown exception isn't caught. We'll report this on the // offending MPI process. /* TEST_FOR_EXCEPTION( info != 0, std::runtime_error, "Failed to insert n=" << n << " global value" << (n != 1 ? "s" : "") << " in row " << MyGlobalRowElements[i] << " of the matrix." ); */ } // for i = 0... // Call FillComplete on the matrix. Since the matrix isn't square, // we have to give FillComplete the domain and range maps, which in // this case are the column resp. row maps. info = A->FillComplete (ColMap, RowMap); /* TEST_FOR_EXCEPTION( info != 0, std::runtime_error, "FillComplete failed with INFO = " << info << "."); */ info = A->OptimizeStorage(); /* TEST_FOR_EXCEPTION( info != 0, std::runtime_error, "OptimizeStorage failed with INFO = " << info << "."); */ } catch (std::runtime_error& e) { // If multiple MPI processes are reporting errors, sometimes // forming the error message as a string and then writing it to // the output stream prevents messages from different processes // from being interleaved. std::ostringstream os; os << "*** Error on MPI process " << MyPID << ": " << e.what(); cerr << os.str() << endl; if (info == 0) info = -1; // All procs will share info later on. } // Variables used for the Block Davidson Method const int nev = 5; const int blockSize = 5; const int numBlocks = 8; const int maxRestarts = 500; const double tol = 1.0e-8; typedef Epetra_MultiVector MV; typedef Epetra_Operator OP; typedef Anasazi::MultiVecTraits<double, Epetra_MultiVector> MVT; // Create an Epetra_MultiVector for an initial vector to start the solver. // Note: This needs to have the same number of columns as the blocksize. // //Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(*Map, blockSize) ); Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(ColMap, blockSize) ); ivec->Random(); // Create the eigenproblem. Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > problem = Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(A, ivec) ); // Inform the eigenproblem that the operator A is symmetric problem->setHermitian(true); // Set the number of eigenvalues requested problem->setNEV( nev ); // Inform the eigenproblem that you are finishing passing it information bool boolret = problem->setProblem(); if (boolret != true) { std::cout<<"Anasazi::BasicEigenproblem::setProblem() returned an error." << std::endl; #ifdef HAVE_MPI MPI_Finalize(); #endif return -1; } // Create parameter list to pass into the solver manager Teuchos::ParameterList anasaziPL; anasaziPL.set( "Which", "LM" ); anasaziPL.set( "Block Size", blockSize ); anasaziPL.set( "Maximum Iterations", 500 ); anasaziPL.set( "Convergence Tolerance", tol ); anasaziPL.set( "Verbosity", Anasazi::Errors+Anasazi::Warnings+Anasazi::TimingDetails+Anasazi::FinalSummary ); // Create the solver manager Anasazi::LOBPCGSolMgr<double, MV, OP> anasaziSolver(problem, anasaziPL); // Solve the problem double start, end; MPI_Barrier(MPI_COMM_WORLD); start = MPI_Wtime(); Anasazi::ReturnType returnCode = anasaziSolver.solve(); MPI_Barrier(MPI_COMM_WORLD); end = MPI_Wtime(); // Get the eigenvalues and eigenvectors from the eigenproblem Anasazi::Eigensolution<double,MV> sol = problem->getSolution(); std::vector<Anasazi::Value<double> > evals = sol.Evals; Teuchos::RCP<MV> evecs = sol.Evecs; // Compute residuals. std::vector<double> normR(sol.numVecs); Teuchos::SerialDenseMatrix<int,double> T(sol.numVecs, sol.numVecs); Epetra_MultiVector tempAevec( ColMap, sol.numVecs ); T.putScalar(0.0); for (int i=0; i<sol.numVecs; i++) { T(i,i) = evals[i].realpart; } A->Apply( *evecs, tempAevec ); MVT::MvTimesMatAddMv( -1.0, *evecs, T, 1.0, tempAevec ); MVT::MvNorm( tempAevec, normR ); if (MyPID == 0) { // Print the results std::cout<<"Solver manager returned " << (returnCode == Anasazi::Converged ? "converged." : "unconverged.") << std::endl; std::cout<<std::endl; std::cout<<"------------------------------------------------------"<<std::endl; std::cout<<std::setw(16)<<"Eigenvalue" <<std::setw(18)<<"Direct Residual" <<std::endl; std::cout<<"------------------------------------------------------"<<std::endl; for (int i=0; i<sol.numVecs; i++) { std::cout<<std::setw(16)<<evals[i].realpart <<std::setw(18)<<normR[i]/evals[i].realpart <<std::endl; } std::cout<<"------------------------------------------------------"<<std::endl; } // Print out the map and matrices //ColMap.Print (out); //A->Print (cout); //RowMap.Print (cout); double time; int iter; if (MyPID==0) { iter = anasaziSolver.getNumIters(); Teuchos::Array<Teuchos::RCP<Teuchos::Time> > timer = anasaziSolver.getTimers(); Teuchos::RCP<Teuchos::Time> _timerSolve = timer[0]; cout << "timerSolve=" << _timerSolve << endl; time = end - start; cout << "time=" << time << endl; ofs << "time=" << time << endl; cout << "iter=" << iter << endl; ofs << "iter=" << iter << endl; } #ifdef HAVE_MPI MPI_Finalize() ; #endif return 0; }
// // Amesos_TestMultiSolver.cpp reads in a matrix in Harwell-Boeing format, // calls one of the sparse direct solvers, using blocked right hand sides // and computes the error and residual. // // TestSolver ignores the Harwell-Boeing right hand sides, creating // random right hand sides instead. // // Amesos_TestMultiSolver can test either A x = b or A^T x = b. // This can be a bit confusing because sparse direct solvers // use compressed column storage - the transpose of Trilinos' // sparse row storage. // // Matrices: // readA - Serial. As read from the file. // transposeA - Serial. The transpose of readA. // serialA - if (transpose) then transposeA else readA // distributedA - readA distributed to all processes // passA - if ( distributed ) then distributedA else serialA // // int Amesos_TestMultiSolver( Epetra_Comm &Comm, char *matrix_file, int numsolves, SparseSolverType SparseSolver, bool transpose, int special, AMESOS_MatrixType matrix_type ) { int iam = Comm.MyPID() ; // int hatever; // if ( iam == 0 ) std::cin >> hatever ; Comm.Barrier(); Epetra_Map * readMap; Epetra_CrsMatrix * readA; Epetra_Vector * readx; Epetra_Vector * readb; Epetra_Vector * readxexact; std::string FileName = matrix_file ; int FN_Size = FileName.size() ; std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size ); std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size ); bool NonContiguousMap = false; if ( LastFiveBytes == ".triU" ) { NonContiguousMap = true; // Call routine to read in unsymmetric Triplet matrix EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, false, Comm, readMap, readA, readx, readb, readxexact, NonContiguousMap ) ); } else { if ( LastFiveBytes == ".triS" ) { NonContiguousMap = true; // Call routine to read in symmetric Triplet matrix EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, true, Comm, readMap, readA, readx, readb, readxexact, NonContiguousMap ) ); } else { if ( LastFourBytes == ".mtx" ) { EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( matrix_file, Comm, readMap, readA, readx, readb, readxexact) ); } else { // Call routine to read in HB problem Trilinos_Util_ReadHb2Epetra( matrix_file, Comm, readMap, readA, readx, readb, readxexact) ; } } } Epetra_CrsMatrix transposeA(Copy, *readMap, 0); Epetra_CrsMatrix *serialA ; if ( transpose ) { assert( CrsMatrixTranspose( readA, &transposeA ) == 0 ); serialA = &transposeA ; } else { serialA = readA ; } // Create uniform distributed map Epetra_Map map(readMap->NumGlobalElements(), 0, Comm); Epetra_Map* map_; if( NonContiguousMap ) { // // map gives us NumMyElements and MyFirstElement; // int NumGlobalElements = readMap->NumGlobalElements(); int NumMyElements = map.NumMyElements(); int MyFirstElement = map.MinMyGID(); std::vector<int> MapMap_( NumGlobalElements ); readMap->MyGlobalElements( &MapMap_[0] ) ; Comm.Broadcast( &MapMap_[0], NumGlobalElements, 0 ) ; map_ = new Epetra_Map( NumGlobalElements, NumMyElements, &MapMap_[MyFirstElement], 0, Comm); } else { map_ = new Epetra_Map( map ) ; } // Create Exporter to distribute read-in matrix and vectors Epetra_Export exporter(*readMap, *map_); Epetra_CrsMatrix A(Copy, *map_, 0); Epetra_RowMatrix * passA = 0; Epetra_MultiVector * passx = 0; Epetra_MultiVector * passb = 0; Epetra_MultiVector * passxexact = 0; Epetra_MultiVector * passresid = 0; Epetra_MultiVector * passtmp = 0; Epetra_MultiVector x(*map_,numsolves); Epetra_MultiVector b(*map_,numsolves); Epetra_MultiVector xexact(*map_,numsolves); Epetra_MultiVector resid(*map_,numsolves); Epetra_MultiVector tmp(*map_,numsolves); Epetra_MultiVector serialx(*readMap,numsolves); Epetra_MultiVector serialb(*readMap,numsolves); Epetra_MultiVector serialxexact(*readMap,numsolves); Epetra_MultiVector serialresid(*readMap,numsolves); Epetra_MultiVector serialtmp(*readMap,numsolves); bool distribute_matrix = ( matrix_type == AMESOS_Distributed ) ; if ( distribute_matrix ) { // // Initialize x, b and xexact to the values read in from the file // A.Export(*serialA, exporter, Add); Comm.Barrier(); assert(A.FillComplete()==0); Comm.Barrier(); passA = &A; passx = &x; passb = &b; passxexact = &xexact; passresid = &resid; passtmp = &tmp; } else { passA = serialA; passx = &serialx; passb = &serialb; passxexact = &serialxexact; passresid = &serialresid; passtmp = &serialtmp; } passxexact->SetSeed(131) ; passxexact->Random(); passx->SetSeed(11231) ; passx->Random(); passb->PutScalar( 0.0 ); passA->Multiply( transpose, *passxexact, *passb ) ; Epetra_MultiVector CopyB( *passb ) ; double Anorm = passA->NormInf() ; SparseDirectTimingVars::SS_Result.Set_Anorm(Anorm) ; Epetra_LinearProblem Problem( (Epetra_RowMatrix *) passA, (Epetra_MultiVector *) passx, (Epetra_MultiVector *) passb ); double max_resid = 0.0; for ( int j = 0 ; j < special+1 ; j++ ) { Epetra_Time TotalTime( Comm ) ; if ( false ) { #ifdef TEST_UMFPACK unused code } else if ( SparseSolver == UMFPACK ) { UmfpackOO umfpack( (Epetra_RowMatrix *) passA, (Epetra_MultiVector *) passx, (Epetra_MultiVector *) passb ) ; umfpack.SetTrans( transpose ) ; umfpack.Solve() ; #endif #ifdef TEST_SUPERLU } else if ( SparseSolver == SuperLU ) { SuperluserialOO superluserial( (Epetra_RowMatrix *) passA, (Epetra_MultiVector *) passx, (Epetra_MultiVector *) passb ) ; superluserial.SetPermc( SuperLU_permc ) ; superluserial.SetTrans( transpose ) ; superluserial.SetUseDGSSV( special == 0 ) ; superluserial.Solve() ; #endif #ifdef HAVE_AMESOS_SLUD } else if ( SparseSolver == SuperLUdist ) { SuperludistOO superludist( Problem ) ; superludist.SetTrans( transpose ) ; EPETRA_CHK_ERR( superludist.Solve( true ) ) ; #endif #ifdef HAVE_AMESOS_SLUD2 } else if ( SparseSolver == SuperLUdist2 ) { Superludist2_OO superludist2( Problem ) ; superludist2.SetTrans( transpose ) ; EPETRA_CHK_ERR( superludist2.Solve( true ) ) ; #endif #ifdef TEST_SPOOLES } else if ( SparseSolver == SPOOLES ) { SpoolesOO spooles( (Epetra_RowMatrix *) passA, (Epetra_MultiVector *) passx, (Epetra_MultiVector *) passb ) ; spooles.SetTrans( transpose ) ; spooles.Solve() ; #endif #ifdef HAVE_AMESOS_DSCPACK } else if ( SparseSolver == DSCPACK ) { Teuchos::ParameterList ParamList ; Amesos_Dscpack dscpack( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( dscpack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( dscpack.Solve( ) ); #endif #ifdef HAVE_AMESOS_UMFPACK } else if ( SparseSolver == UMFPACK ) { Teuchos::ParameterList ParamList ; Amesos_Umfpack umfpack( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( umfpack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( umfpack.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( umfpack.Solve( ) ); #endif #ifdef HAVE_AMESOS_KLU } else if ( SparseSolver == KLU ) { Teuchos::ParameterList ParamList ; Amesos_Klu klu( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( klu.SetParameters( ParamList ) ); EPETRA_CHK_ERR( klu.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( klu.SymbolicFactorization( ) ); EPETRA_CHK_ERR( klu.NumericFactorization( ) ); EPETRA_CHK_ERR( klu.Solve( ) ); #endif #ifdef HAVE_AMESOS_PARAKLETE } else if ( SparseSolver == PARAKLETE ) { Teuchos::ParameterList ParamList ; Amesos_Paraklete paraklete( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( paraklete.SetParameters( ParamList ) ); EPETRA_CHK_ERR( paraklete.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( paraklete.SymbolicFactorization( ) ); EPETRA_CHK_ERR( paraklete.NumericFactorization( ) ); EPETRA_CHK_ERR( paraklete.Solve( ) ); #endif #ifdef HAVE_AMESOS_SLUS } else if ( SparseSolver == SuperLU ) { Epetra_SLU superluserial( &Problem ) ; EPETRA_CHK_ERR( superluserial.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( superluserial.SymbolicFactorization( ) ); EPETRA_CHK_ERR( superluserial.NumericFactorization( ) ); EPETRA_CHK_ERR( superluserial.Solve( ) ); #endif #ifdef HAVE_AMESOS_LAPACK } else if ( SparseSolver == LAPACK ) { Teuchos::ParameterList ParamList ; ParamList.set( "MaxProcs", -3 ); Amesos_Lapack lapack( Problem ) ; EPETRA_CHK_ERR( lapack.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( lapack.SymbolicFactorization( ) ); EPETRA_CHK_ERR( lapack.NumericFactorization( ) ); EPETRA_CHK_ERR( lapack.Solve( ) ); #endif #ifdef HAVE_AMESOS_TAUCS } else if ( SparseSolver == TAUCS ) { Teuchos::ParameterList ParamList ; Amesos_Taucs taucs( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( taucs.SetParameters( ParamList ) ); EPETRA_CHK_ERR( taucs.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( taucs.SymbolicFactorization( ) ); EPETRA_CHK_ERR( taucs.NumericFactorization( ) ); EPETRA_CHK_ERR( taucs.Solve( ) ); #endif #ifdef HAVE_AMESOS_PARDISO } else if ( SparseSolver == PARDISO ) { Teuchos::ParameterList ParamList ; Amesos_Pardiso pardiso( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( pardiso.SetParameters( ParamList ) ); EPETRA_CHK_ERR( pardiso.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( pardiso.SymbolicFactorization( ) ); EPETRA_CHK_ERR( pardiso.NumericFactorization( ) ); EPETRA_CHK_ERR( pardiso.Solve( ) ); #endif #ifdef HAVE_AMESOS_PARKLETE } else if ( SparseSolver == PARKLETE ) { Teuchos::ParameterList ParamList ; Amesos_Parklete parklete( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( parklete.SetParameters( ParamList ) ); EPETRA_CHK_ERR( parklete.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( parklete.SymbolicFactorization( ) ); EPETRA_CHK_ERR( parklete.NumericFactorization( ) ); EPETRA_CHK_ERR( parklete.Solve( ) ); #endif #ifdef HAVE_AMESOS_MUMPS } else if ( SparseSolver == MUMPS ) { Teuchos::ParameterList ParamList ; Amesos_Mumps mumps( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( mumps.SetParameters( ParamList ) ); EPETRA_CHK_ERR( mumps.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( mumps.SymbolicFactorization( ) ); EPETRA_CHK_ERR( mumps.NumericFactorization( ) ); EPETRA_CHK_ERR( mumps.Solve( ) ); #endif #ifdef HAVE_AMESOS_SCALAPACK } else if ( SparseSolver == SCALAPACK ) { Teuchos::ParameterList ParamList ; Amesos_Scalapack scalapack( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( scalapack.SetParameters( ParamList ) ); EPETRA_CHK_ERR( scalapack.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( scalapack.SymbolicFactorization( ) ); EPETRA_CHK_ERR( scalapack.NumericFactorization( ) ); EPETRA_CHK_ERR( scalapack.Solve( ) ); #endif #ifdef HAVE_AMESOS_SUPERLUDIST } else if ( SparseSolver == SUPERLUDIST ) { Teuchos::ParameterList ParamList ; Amesos_Superludist superludist( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( superludist.SetParameters( ParamList ) ); EPETRA_CHK_ERR( superludist.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( superludist.SymbolicFactorization( ) ); EPETRA_CHK_ERR( superludist.NumericFactorization( ) ); EPETRA_CHK_ERR( superludist.Solve( ) ); #endif #ifdef HAVE_AMESOS_SUPERLU } else if ( SparseSolver == SUPERLU ) { Teuchos::ParameterList ParamList ; Amesos_Superlu superlu( Problem ) ; ParamList.set( "MaxProcs", -3 ); EPETRA_CHK_ERR( superlu.SetParameters( ParamList ) ); EPETRA_CHK_ERR( superlu.SetUseTranspose( transpose ) ); EPETRA_CHK_ERR( superlu.SymbolicFactorization( ) ); EPETRA_CHK_ERR( superlu.NumericFactorization( ) ); EPETRA_CHK_ERR( superlu.Solve( ) ); #endif #ifdef TEST_SPOOLESSERIAL } else if ( SparseSolver == SPOOLESSERIAL ) { SpoolesserialOO spoolesserial( (Epetra_RowMatrix *) passA, (Epetra_MultiVector *) passx, (Epetra_MultiVector *) passb ) ; spoolesserial.Solve() ; #endif } else { SparseDirectTimingVars::log_file << "Solver not implemented yet" << std::endl ; std::cerr << "\n\n#################### Requested solver not available (Or not tested with blocked RHS) on this platform #####################\n" << std::endl ; } SparseDirectTimingVars::SS_Result.Set_Total_Time( TotalTime.ElapsedTime() ); // SparseDirectTimingVars::SS_Result.Set_First_Time( 0.0 ); // SparseDirectTimingVars::SS_Result.Set_Middle_Time( 0.0 ); // SparseDirectTimingVars::SS_Result.Set_Last_Time( 0.0 ); // // Compute the error = norm(xcomp - xexact ) // std::vector <double> error(numsolves) ; double max_error = 0.0; passresid->Update(1.0, *passx, -1.0, *passxexact, 0.0); passresid->Norm2(&error[0]); for ( int i = 0 ; i< numsolves; i++ ) if ( error[i] > max_error ) max_error = error[i] ; SparseDirectTimingVars::SS_Result.Set_Error(max_error) ; // passxexact->Norm2(&error[0] ) ; // passx->Norm2(&error ) ; // // Compute the residual = norm(Ax - b) // std::vector <double> residual(numsolves) ; passtmp->PutScalar(0.0); passA->Multiply( transpose, *passx, *passtmp); passresid->Update(1.0, *passtmp, -1.0, *passb, 0.0); // passresid->Update(1.0, *passtmp, -1.0, CopyB, 0.0); passresid->Norm2(&residual[0]); for ( int i = 0 ; i< numsolves; i++ ) if ( residual[i] > max_resid ) max_resid = residual[i] ; SparseDirectTimingVars::SS_Result.Set_Residual(max_resid) ; std::vector <double> bnorm(numsolves); passb->Norm2( &bnorm[0] ) ; SparseDirectTimingVars::SS_Result.Set_Bnorm(bnorm[0]) ; std::vector <double> xnorm(numsolves); passx->Norm2( &xnorm[0] ) ; SparseDirectTimingVars::SS_Result.Set_Xnorm(xnorm[0]) ; if ( false && iam == 0 ) { std::cout << " Amesos_TestMutliSolver.cpp " << std::endl ; for ( int i = 0 ; i< numsolves && i < 10 ; i++ ) { std::cout << "i=" << i << " error = " << error[i] << " xnorm = " << xnorm[i] << " residual = " << residual[i] << " bnorm = " << bnorm[i] << std::endl ; } std::cout << std::endl << " max_resid = " << max_resid ; std::cout << " max_error = " << max_error << std::endl ; std::cout << " Get_residual() again = " << SparseDirectTimingVars::SS_Result.Get_Residual() << std::endl ; } } delete readA; delete readx; delete readb; delete readxexact; delete readMap; delete map_; Comm.Barrier(); return 0 ; }
// ============================================================================ void EpetraExt::XMLWriter:: Write(const std::string& Label, const Epetra_Map& Map) { TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error, "No file has been opened"); int NumGlobalElements = Map.NumGlobalElements(); int* MyGlobalElements = Map.MyGlobalElements(); if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "<Map Label=\"" << Label << "\" NumElements=\"" << NumGlobalElements << '"' << " IndexBase=\"" << Map.IndexBase() << '"' << " NumProc=\"" << Comm_.NumProc() << '"'; of.close(); } for (int iproc = 0; iproc < Comm_.NumProc(); ++iproc) { if (iproc == Comm_.MyPID()) { std::ofstream of(FileName_.c_str(), std::ios::app); of << " ElementsOnProc" << iproc << "=\"" << Map.NumMyElements() << '"'; of.close(); } Comm_.Barrier(); } if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << '>' << std::endl; of.close(); } for (int iproc = 0; iproc < Comm_.NumProc(); iproc++) { if (iproc == Comm_.MyPID()) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "<Proc ID=\"" << Comm_.MyPID() << "\">" << std::endl; for (int i = 0; i < Map.NumMyElements(); ++i) { of << MyGlobalElements[i] << std::endl; } of << "</Proc>" << std::endl; of.close(); } Comm_.Barrier(); } if (Comm_.MyPID() == 0) { std::ofstream of(FileName_.c_str(), std::ios::app); of << "</Map>" << std::endl; of.close(); } }
int main (int argc, char *argv[]) { using namespace Anasazi; using Teuchos::RCP; using Teuchos::rcp; using std::endl; #ifdef HAVE_MPI // Initialize MPI MPI_Init (&argc, &argv); #endif // HAVE_MPI // Create an Epetra communicator #ifdef HAVE_MPI Epetra_MpiComm Comm (MPI_COMM_WORLD); #else Epetra_SerialComm Comm; #endif // HAVE_MPI // Create an Anasazi output manager BasicOutputManager<double> printer; printer.stream(Errors) << Anasazi_Version() << std::endl << std::endl; // Get the sorting std::string from the command line std::string which ("LM"); Teuchos::CommandLineProcessor cmdp (false, true); cmdp.setOption("sort", &which, "Targetted eigenvalues (SM or LM)."); if (cmdp.parse (argc, argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) { #ifdef HAVE_MPI MPI_Finalize (); #endif // HAVE_MPI return -1; } // Dimension of the matrix // // Discretization points in any one direction. const int nx = 10; // Size of matrix nx*nx const int NumGlobalElements = nx*nx; // Construct a Map that puts approximately the same number of // equations on each process. Epetra_Map Map (NumGlobalElements, 0, Comm); // Get update list and number of local equations from newly created Map. int NumMyElements = Map.NumMyElements (); std::vector<int> MyGlobalElements (NumMyElements); Map.MyGlobalElements (&MyGlobalElements[0]); // Create an integer vector NumNz that is used to build the Petra // matrix. NumNz[i] is the number of OFF-DIAGONAL terms for the // i-th global equation on this process. std::vector<int> NumNz (NumMyElements); /* We are building a matrix of block structure: | T -I | |-I T -I | | -I T | | ... -I| | -I T| where each block is dimension nx by nx and the matrix is on the order of nx*nx. The block T is a tridiagonal matrix. */ for (int i=0; i<NumMyElements; ++i) { if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1 || MyGlobalElements[i] == nx-1 || MyGlobalElements[i] == nx*(nx-1) ) { NumNz[i] = 3; } else if (MyGlobalElements[i] < nx || MyGlobalElements[i] > nx*(nx-1) || MyGlobalElements[i]%nx == 0 || (MyGlobalElements[i]+1)%nx == 0) { NumNz[i] = 4; } else { NumNz[i] = 5; } } // Create an Epetra_Matrix RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, &NumNz[0])); // Compute coefficients for discrete convection-diffution operator const double one = 1.0; std::vector<double> Values(4); std::vector<int> Indices(4); double rho = 0.0; double h = one /(nx+1); double h2 = h*h; double c = 5.0e-01*rho/ h; Values[0] = -one/h2 - c; Values[1] = -one/h2 + c; Values[2] = -one/h2; Values[3]= -one/h2; double diag = 4.0 / h2; int NumEntries; for (int i=0; i<NumMyElements; ++i) { if (MyGlobalElements[i]==0) { Indices[0] = 1; Indices[1] = nx; NumEntries = 2; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if (MyGlobalElements[i] == nx*(nx-1)) { Indices[0] = nx*(nx-1)+1; Indices[1] = nx*(nx-2); NumEntries = 2; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if (MyGlobalElements[i] == nx-1) { Indices[0] = nx-2; NumEntries = 1; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); Indices[0] = 2*nx-1; info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if (MyGlobalElements[i] == NumGlobalElements-1) { Indices[0] = NumGlobalElements-2; NumEntries = 1; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); Indices[0] = nx*(nx-1)-1; info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if (MyGlobalElements[i] < nx) { Indices[0] = MyGlobalElements[i]-1; Indices[1] = MyGlobalElements[i]+1; Indices[2] = MyGlobalElements[i]+nx; NumEntries = 3; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if (MyGlobalElements[i] > nx*(nx-1)) { Indices[0] = MyGlobalElements[i]-1; Indices[1] = MyGlobalElements[i]+1; Indices[2] = MyGlobalElements[i]-nx; NumEntries = 3; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if (MyGlobalElements[i]%nx == 0) { Indices[0] = MyGlobalElements[i]+1; Indices[1] = MyGlobalElements[i]-nx; Indices[2] = MyGlobalElements[i]+nx; NumEntries = 3; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else if ((MyGlobalElements[i]+1)%nx == 0) { Indices[0] = MyGlobalElements[i]-nx; Indices[1] = MyGlobalElements[i]+nx; NumEntries = 2; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); Indices[0] = MyGlobalElements[i]-1; NumEntries = 1; info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } else { Indices[0] = MyGlobalElements[i]-1; Indices[1] = MyGlobalElements[i]+1; Indices[2] = MyGlobalElements[i]-nx; Indices[3] = MyGlobalElements[i]+nx; NumEntries = 4; int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } // Put in the diagonal entry int info = A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, &MyGlobalElements[i]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "InsertGlobalValues returned info = " << info << " != 0." ); } // Finish up int info = A->FillComplete (); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "A->FillComplete() returned info = " << info << " != 0." ); A->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks // Create a identity matrix for the temporary mass matrix RCP<Epetra_CrsMatrix> M = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, 1)); for (int i=0; i<NumMyElements; i++) { Values[0] = one; Indices[0] = i; NumEntries = 1; info = M->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "M->InsertGlobalValues() returned info = " << info << " != 0." ); } // Finish up info = M->FillComplete (); TEUCHOS_TEST_FOR_EXCEPTION (info != 0, std::runtime_error, "M->FillComplete() returned info = " << info << " != 0." ); M->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks //************************************ // Call the LOBPCG solver manager //*********************************** // // Variables used for the LOBPCG Method const int nev = 10; const int blockSize = 5; const int maxIters = 500; const double tol = 1.0e-8; typedef Epetra_MultiVector MV; typedef Epetra_Operator OP; typedef MultiVecTraits<double, Epetra_MultiVector> MVT; // Create an Epetra_MultiVector for an initial vector to start the // solver. Note: This needs to have the same number of columns as // the blocksize. RCP<Epetra_MultiVector> ivec = rcp (new Epetra_MultiVector (Map, blockSize)); ivec->Random (); // fill the initial vector with random values // Create the eigenproblem. RCP<BasicEigenproblem<double, MV, OP> > MyProblem = rcp (new BasicEigenproblem<double, MV, OP> (A, ivec)); // Inform the eigenproblem that the operator A is symmetric MyProblem->setHermitian (true); // Set the number of eigenvalues requested MyProblem->setNEV (nev); // Tell the eigenproblem that you are finishing passing it information. const bool success = MyProblem->setProblem (); if (! success) { printer.print (Errors, "Anasazi::BasicEigenproblem::setProblem() reported an error.\n"); #ifdef HAVE_MPI MPI_Finalize (); #endif // HAVE_MPI return -1; } // Create parameter list to pass into the solver manager Teuchos::ParameterList MyPL; MyPL.set ("Which", which); MyPL.set ("Block Size", blockSize); MyPL.set ("Maximum Iterations", maxIters); MyPL.set ("Convergence Tolerance", tol); MyPL.set ("Full Ortho", true); MyPL.set ("Use Locking", true); // Create the solver manager LOBPCGSolMgr<double, MV, OP> MySolverMan (MyProblem, MyPL); // Solve the problem ReturnType returnCode = MySolverMan.solve (); // Get the eigenvalues and eigenvectors from the eigenproblem Eigensolution<double,MV> sol = MyProblem->getSolution (); std::vector<Value<double> > evals = sol.Evals; RCP<MV> evecs = sol.Evecs; // Compute residuals. std::vector<double> normR (sol.numVecs); if (sol.numVecs > 0) { Teuchos::SerialDenseMatrix<int,double> T (sol.numVecs, sol.numVecs); Epetra_MultiVector tempAevec (Map, sol.numVecs ); T.putScalar (0.0); for (int i = 0; i < sol.numVecs; ++i) { T(i,i) = evals[i].realpart; } A->Apply (*evecs, tempAevec); MVT::MvTimesMatAddMv (-1.0, *evecs, T, 1.0, tempAevec); MVT::MvNorm (tempAevec, normR); } // Print the results std::ostringstream os; os.setf (std::ios_base::right, std::ios_base::adjustfield); os << "Solver manager returned " << (returnCode == Converged ? "converged." : "unconverged.") << endl; os << endl; os << "------------------------------------------------------" << endl; os << std::setw(16) << "Eigenvalue" << std::setw(18) << "Direct Residual" << endl; os << "------------------------------------------------------" << endl; for (int i = 0; i < sol.numVecs; ++i) { os << std::setw(16) << evals[i].realpart << std::setw(18) << normR[i] / evals[i].realpart << endl; } os << "------------------------------------------------------" << endl; printer.print (Errors, os.str ()); #ifdef HAVE_MPI MPI_Finalize (); #endif // HAVE_MPI return 0; }
int main(int argc, char *argv[]) { int ierr=0, returnierr=0; #ifdef EPETRA_MPI MPI_Init(&argc,&argv); Epetra_MpiComm Comm(MPI_COMM_WORLD); #else Epetra_SerialComm Comm; #endif bool verbose = false; // Check if we should print results to standard out if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true; if (!verbose) { Comm.SetTracebackMode(0); // This should shut down any error traceback reporting } int MyPID = Comm.MyPID(); int NumProc = Comm.NumProc(); if (verbose && MyPID==0) cout << Epetra_Version() << endl << endl; if (verbose) cout << Comm << endl; bool verbose1 = verbose; if (verbose) verbose = (MyPID==0); int NumMyElements = 10000; int NumMyElements1 = NumMyElements; // Used for local map int NumGlobalElements = NumMyElements*NumProc+EPETRA_MIN(NumProc,3); if (MyPID < 3) NumMyElements++; int IndexBase = 0; bool DistributedGlobal = (NumGlobalElements>NumMyElements); Epetra_Map* Map; // Test exceptions if (verbose) cout << "*******************************************************************************************" << endl << " Testing Exceptions (Expect error messages if EPETRA_NO_ERROR_REPORTS is not defined" << endl << "*******************************************************************************************" << endl << endl << endl; try { if (verbose) cout << "Checking Epetra_Map(-2, IndexBase, Comm)" << endl; Map = new Epetra_Map(-2, IndexBase, Comm); } catch (int Error) { if (Error!=-1) { if (Error!=0) { EPETRA_TEST_ERR(Error,returnierr); if (verbose) cout << "Error code should be -1" << endl; } else { cout << "Error code = " << Error << "Should be -1" << endl; returnierr+=1; } } else if (verbose) cout << "Checked OK\n\n" << endl; } try { if (verbose) cout << "Checking Epetra_Map(2, 3, IndexBase, Comm)" << endl; Map = new Epetra_Map(2, 3, IndexBase, Comm); } catch (int Error) { if (Error!=-4) { if (Error!=0) { EPETRA_TEST_ERR(Error,returnierr); if (verbose) cout << "Error code should be -4" << endl; } else { cout << "Error code = " << Error << "Should be -4" << endl; returnierr+=1; } } else if (verbose) cout << "Checked OK\n\n" << endl; } if (verbose) cerr << flush; if (verbose) cout << flush; Comm.Barrier(); if (verbose) cout << endl << endl << "*******************************************************************************************" << endl << " Testing valid constructor now......................................................" << endl << "*******************************************************************************************" << endl << endl << endl; // Test Epetra-defined uniform linear distribution constructor Map = new Epetra_Map(NumGlobalElements, IndexBase, Comm); if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, IndexBase, Comm)" << endl; ierr = checkmap(*Map, NumGlobalElements, NumMyElements, 0, IndexBase, Comm, DistributedGlobal); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; delete Map; // Test User-defined linear distribution constructor Map = new Epetra_Map(NumGlobalElements, NumMyElements, IndexBase, Comm); if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, NumMyElements, IndexBase, Comm)" << endl; ierr = checkmap(*Map, NumGlobalElements, NumMyElements, 0, IndexBase, Comm, DistributedGlobal); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; delete Map; // Test User-defined arbitrary distribution constructor // Generate Global Element List. Do in reverse for fun! int * MyGlobalElements = new int[NumMyElements]; int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase; if (Comm.MyPID()>2) MaxMyGID+=3; for (int i = 0; i<NumMyElements; i++) MyGlobalElements[i] = MaxMyGID-i; Map = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm); if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm)" << endl; ierr = checkmap(*Map, NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm, DistributedGlobal); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; // Test Copy constructor Epetra_Map* Map1 = new Epetra_Map(*Map); // Test SameAs() method bool same = Map1->SameAs(*Map); EPETRA_TEST_ERR(!(same==true),ierr);// should return true since Map1 is a copy of Map Epetra_BlockMap* Map2 = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm); same = Map2->SameAs(*Map); EPETRA_TEST_ERR(!(same==true),ierr); // Map and Map2 were created with the same sets of parameters delete Map2; // now test SameAs() on a map that is different Map2 = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase-1, Comm); same = Map2->SameAs(*Map); EPETRA_TEST_ERR(!(same==false),ierr); // IndexBases are different delete Map2; // Back to testing copy constructor if (verbose) cout << "Checking Epetra_Map(*Map)" << endl; ierr = checkmap(*Map1, NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm, DistributedGlobal); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; Epetra_Map* SmallMap = 0; if (verbose1) { // Build a small map for test cout. Use 10 elements from current map int* MyEls = Map->MyGlobalElements(); int IndBase = Map->IndexBase(); int MyLen = EPETRA_MIN(10+Comm.MyPID(),Map->NumMyElements()); SmallMap = new Epetra_Map(-1, MyLen, MyEls, IndBase, Comm); } delete [] MyGlobalElements; delete Map; delete Map1; // Test reference-counting in Epetra_Map if (verbose) cout << "Checking Epetra_Map reference counting" << endl; ierr = checkMapDataClass(Comm, verbose); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; // Test LocalMap constructor Epetra_LocalMap* LocalMap = new Epetra_LocalMap(NumMyElements1, IndexBase, Comm); if (verbose) cout << "Checking Epetra_LocalMap(NumMyElements1, IndexBase, Comm)" << endl; ierr = checkmap(*LocalMap, NumMyElements1, NumMyElements1, 0, IndexBase, Comm, false); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; // Test Copy constructor Epetra_LocalMap* LocalMap1 = new Epetra_LocalMap(*LocalMap); if (verbose) cout << "Checking Epetra_LocalMap(*LocalMap)" << endl; ierr = checkmap(*LocalMap1, NumMyElements1, NumMyElements1, 0, IndexBase, Comm, false); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; delete LocalMap1; delete LocalMap; // Test reference-counting in Epetra_LocalMap if (verbose) cout << "Checking Epetra_LocalMap reference counting" << endl; ierr = checkLocalMapDataClass(Comm, verbose); EPETRA_TEST_ERR(ierr,returnierr); if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl; // Test output if (verbose1) { if (verbose) cout << "Test ostream << operator" << endl << flush; cout << *SmallMap; delete SmallMap; } #ifdef EPETRA_MPI MPI_Finalize(); #endif return returnierr; }