//================================================================
//================================================================
// RN_20091215: This needs to be called only once per time step
// in the beginning to set up the problem.
//================================================================
void FC_FUNC(inittrilinos,INITTRILINOS) (int& bandwidth, int& mySize,
int* myIndicies, double* myX, double* myY, double* myZ,
int* mpi_comm_f) {
// mpi_comm_f: CISM's fortran mpi communicator
#ifdef GLIMMER_MPI
// Make sure the MPI_Init in Fortran is recognized by C++.
// We used to call an extra MPI_Init if (!flag), but the behavior of doing so is uncertain,
// especially if CISM's MPI communicator is a subset of MPI_COMM_WORLD (as can be the case in CESM).
// Thus, for now, we die with an error message if C++ perceives MPI to be uninitialized.
// If this causes problems (e.g., if certain MPI implementations seem not to recognize
// that MPI has already been initialized), then we will revisit how to handle this.
int flag;
MPI_Initialized(&flag);
if (!flag) {
cout << "ERROR in inittrilinos: MPI not initialized according to C++ code" << endl;
exit(1);
}
MPI_Comm mpi_comm_c = MPI_Comm_f2c(*mpi_comm_f);
Epetra_MpiComm comm(mpi_comm_c);
Teuchos::MpiComm<int> tcomm(Teuchos::opaqueWrapper(mpi_comm_c));
#else
Epetra_SerialComm comm;
Teuchos::SerialComm<int> tcomm;
#endif
Teuchos::RCP<const Epetra_Map> rowMap =
Teuchos::rcp(new Epetra_Map(-1,mySize,myIndicies,1,comm) );
TEUCHOS_TEST_FOR_EXCEPTION(!rowMap->UniqueGIDs(), std::logic_error,
"Error: inittrilinos, myIndices array needs to have Unique entries"
<< " across all processor.");
// Diagnostic output for partitioning
int minSize, maxSize;
comm.MinAll(&mySize, &minSize, 1);
comm.MaxAll(&mySize, &maxSize, 1);
if (comm.MyPID()==0)
cout << "\nPartition Info in init_trilinos: Total nodes = " << rowMap->NumGlobalElements()
<< " Max = " << maxSize << " Min = " << minSize
<< " Ave = " << rowMap->NumGlobalElements() / comm.NumProc() << endl;
soln = Teuchos::rcp(new Epetra_Vector(*rowMap));
// Read parameter list once
try {
pl = Teuchos::rcp(new Teuchos::ParameterList("Trilinos Options"));
Teuchos::updateParametersFromXmlFileAndBroadcast("trilinosOptions.xml", pl.ptr(), tcomm);
Teuchos::ParameterList validPL("Valid List");;
validPL.sublist("Stratimikos"); validPL.sublist("Piro");
pl->validateParameters(validPL, 0);
}
catch (std::exception& e) {
cout << "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
<< e.what() << "\nExiting: Invalid trilinosOptions.xml file."
<< "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" << endl;
exit(1);
}
catch (...) {
cout << "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
<< "\nExiting: Invalid trilinosOptions.xml file."
<< "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" << endl;
exit(1);
}
try {
// Set the coordinate position of the nodes for ML for repartitioning (important for #procs > 100s)
if (pl->sublist("Stratimikos").isParameter("Preconditioner Type")) {
if ("ML" == pl->sublist("Stratimikos").get<string>("Preconditioner Type")) {
Teuchos::ParameterList& mlList =
pl->sublist("Stratimikos").sublist("Preconditioner Types").sublist("ML").sublist("ML Settings");
mlList.set("x-coordinates",myX);
mlList.set("y-coordinates",myY);
mlList.set("z-coordinates",myZ);
mlList.set("PDE equations", 1);
}
}
out = Teuchos::VerboseObjectBase::getDefaultOStream();
// Reset counters every time step: can remove these lines to have averages over entire run
linearSolveIters_total = 0;
linearSolveCount=0;
linearSolveSuccessCount = 0;
// Create an interface that holds a CrsMatrix instance and some useful methods.
interface = Teuchos::rcp(new TrilinosMatrix_Interface(rowMap, bandwidth, comm));
Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
linearSolverBuilder.setParameterList(Teuchos::sublist(pl, "Stratimikos"));
lowsFactory = linearSolverBuilder.createLinearSolveStrategy("");
lowsFactory->setOStream(out);
lowsFactory->setVerbLevel(Teuchos::VERB_LOW);
lows=Teuchos::null;
thyraOper=Teuchos::null;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (!success) exit(1);
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
// define an Epetra communicator
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
// get the proc ID of this process
int MyPID = Comm.MyPID();
// get the total number of processes
int NumProc = Comm.NumProc();
// output some information to std output
cout << Comm << endl;
// ======================== //
// now some basic MPI calls //
// ------------------------ //
int ivalue;
double dvalue, dvalue2;
double* dvalues; dvalues = new double[NumProc];
double* dvalues2; dvalues2 = new double[NumProc];
int root = 0;
// equivalent to MPI_Barrier
Comm.Barrier();
if (MyPID == root) dvalue = 12.0;
// On input, the root processor contains the list of values
// (in this case, a single value). On exit, all processes will
// have he same list of values. Note that all values must be allocated
// vefore the broadcast
// equivalent to MPI_Broadcast
Comm.Broadcast(&dvalue, 1, root);
// as before, but with integer values. As C++ can bind to the appropriate
// interface based on argument typing, the type of data is not required.
Comm.Broadcast(&ivalue, 1, root);
// equivalent MPI_Allgather
Comm.GatherAll(dvalues, dvalues2, 1);
// equivalent to MPI_Allreduce with MPI_SUM
dvalue = 1.0*MyPID;
Comm.SumAll( &dvalue, dvalues, 1);
// equivalent to MPI_Allreduce with MPI_SUM
Comm.MaxAll( &dvalue, dvalues, 1);
// equiavant to MPI_Scan with MPI_SUM
dvalue = 1.0 * MyPID;
Comm.ScanSum(&dvalue, &dvalue2, 1);
cout << "On proc " << MyPID << " dvalue2 = " << dvalue2 << endl;
delete[] dvalues;
delete[] dvalues2;
// ======================= //
// Finalize MPI and return //
// ----------------------- //
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return( EXIT_SUCCESS );
} /* main */
