size_t removeUndesiredEdges(
const RCP<const Environment> &env,
int myRank,
bool removeSelfEdges,
bool removeOffProcessEdges,
bool removeOffGroupEdges,
ArrayView<const typename InputTraits<User>::zgid_t> &gids,
ArrayView<const typename InputTraits<User>::zgid_t> &gidNbors,
ArrayView<const int> &procIds,
ArrayView<StridedData<typename InputTraits<User>::lno_t,
typename InputTraits<User>::scalar_t> > &edgeWeights,
ArrayView<const typename InputTraits<User>::lno_t> &offsets,
ArrayRCP<const typename InputTraits<User>::zgid_t> &newGidNbors, // out
typename InputTraits<User>::scalar_t **&newWeights, // out
ArrayRCP<const typename InputTraits<User>::lno_t> &newOffsets) // out
{
typedef typename InputTraits<User>::zgid_t zgid_t;
typedef typename InputTraits<User>::scalar_t scalar_t;
typedef typename InputTraits<User>::lno_t lno_t;
size_t numKeep = 0;
size_t numVtx = offsets.size() - 1;
size_t numNbors = gidNbors.size();
env->localInputAssertion(__FILE__, __LINE__, "need more input",
(!removeSelfEdges ||
gids.size() >=
static_cast<typename ArrayView<const zgid_t>::size_type>(numVtx))
&&
(!removeOffProcessEdges ||
procIds.size() >=
static_cast<typename ArrayView<const int>::size_type>(numNbors)) &&
(!removeOffGroupEdges ||
procIds.size() >=
static_cast<typename ArrayView<const int>::size_type>(numNbors)),
BASIC_ASSERTION);
// initialize edge weight array
newWeights = NULL;
int eDim = edgeWeights.size();
// count desired edges
lno_t *offs = new lno_t [numVtx + 1];
env->localMemoryAssertion(__FILE__, __LINE__, numVtx+1, offs);
for (size_t i = 0; i < numVtx+1; i++) offs[i] = 0;
ArrayRCP<const lno_t> offArray = arcp(offs, 0, numVtx+1, true);
const lno_t *allOffs = offsets.getRawPtr();
const zgid_t *allIds = gidNbors.getRawPtr();
const zgid_t *vtx = NULL;
const int *proc = NULL;
if (gids.size() > 0)
vtx = gids.getRawPtr();
if (procIds.size() > 0)
proc = procIds.getRawPtr();
offs[0] = 0;
for (size_t i=0; i < numVtx; i++){
offs[i+1] = 0;
zgid_t vid = vtx ? vtx[i] : zgid_t(0);
for (lno_t j=allOffs[i]; j < allOffs[i+1]; j++){
int owner = proc ? proc[j] : 0;
bool keep = (!removeSelfEdges || vid != allIds[j]) &&
(!removeOffProcessEdges || owner == myRank) &&
(!removeOffGroupEdges || owner >= 0);
if (keep)
offs[i+1]++;
}
}
// from counters to offsets
for (size_t i=1; i < numVtx; i++)
offs[i+1] += offs[i];
numKeep = offs[numVtx];
// do we need a new neighbor list?
if (numNbors == numKeep){
newGidNbors = Teuchos::arcpFromArrayView(gidNbors);
newOffsets = Teuchos::arcpFromArrayView(offsets);
return numNbors;
}
else if (numKeep == 0){
newGidNbors = ArrayRCP<const zgid_t>(Teuchos::null);
newOffsets = offArray;
return 0;
}
// Build the subset neighbor lists (id, weight, and offset).
zgid_t *newGids = new zgid_t [numKeep];
env->localMemoryAssertion(__FILE__, __LINE__, numKeep, newGids);
newGidNbors = arcp(newGids, 0, numKeep, true);
newOffsets = offArray;
if (eDim > 0){
newWeights = new scalar_t * [eDim];
env->localMemoryAssertion(__FILE__, __LINE__, eDim, newWeights);
if (numKeep) {
for (int w=0; w < eDim; w++){
newWeights[w] = new scalar_t [numKeep];
env->localMemoryAssertion(__FILE__, __LINE__, numKeep, newWeights[w]);
}
}
else {
for (int w=0; w < eDim; w++)
newWeights[w] = NULL;
}
}
size_t next = 0;
for (size_t i=0; i < numVtx && next < numKeep; i++){
zgid_t vid = vtx ? vtx[i] : zgid_t(0);
for (lno_t j=allOffs[i]; j < allOffs[i+1]; j++){
int owner = proc ? proc[j] : 0;
bool keep = (!removeSelfEdges || vid != allIds[j]) &&
(!removeOffProcessEdges || owner == myRank) &&
(!removeOffGroupEdges || owner >= 0);
if (keep){
newGids[next] = allIds[j];
for (int w=0; w < eDim; w++){
newWeights[w][next] = edgeWeights[w][j];
}
next++;
if (next == numKeep)
break;
} // if (keep)
}
}
return numKeep;
}
void
Albany::SamplingBasedScalarResponseFunction::
evaluateSGGradient(
const double current_time,
const Stokhos::EpetraVectorOrthogPoly* sg_xdot,
const Stokhos::EpetraVectorOrthogPoly* sg_xdotdot,
const Stokhos::EpetraVectorOrthogPoly& sg_x,
const Teuchos::Array<ParamVec>& p,
const Teuchos::Array<int>& sg_p_index,
const Teuchos::Array< Teuchos::Array<SGType> >& sg_p_vals,
ParamVec* deriv_p,
Stokhos::EpetraVectorOrthogPoly* sg_g,
Stokhos::EpetraMultiVectorOrthogPoly* sg_dg_dx,
Stokhos::EpetraMultiVectorOrthogPoly* sg_dg_dxdot,
Stokhos::EpetraMultiVectorOrthogPoly* sg_dg_dxdotdot,
Stokhos::EpetraMultiVectorOrthogPoly* sg_dg_dp)
{
RCP<const Epetra_BlockMap> x_map = sg_x.coefficientMap();
RCP<Epetra_Vector> xdot;
if (sg_xdot != NULL)
xdot = rcp(new Epetra_Vector(*x_map));
RCP<Epetra_Vector> xdotdot;
if (sg_xdotdot != NULL)
xdotdot = rcp(new Epetra_Vector(*x_map));
Epetra_Vector x(*x_map);
Teuchos::Array<ParamVec> pp = p;
RCP<Epetra_Vector> g;
if (sg_g != NULL) {
sg_g->init(0.0);
g = rcp(new Epetra_Vector(*(sg_g->coefficientMap())));
}
RCP<Epetra_MultiVector> dg_dx;
if (sg_dg_dx != NULL) {
sg_dg_dx->init(0.0);
dg_dx = rcp(new Epetra_MultiVector(*(sg_dg_dx->coefficientMap()),
sg_dg_dx->numVectors()));
}
RCP<Epetra_MultiVector> dg_dxdot;
if (sg_dg_dxdot != NULL) {
sg_dg_dxdot->init(0.0);
dg_dxdot = rcp(new Epetra_MultiVector(*(sg_dg_dxdot->coefficientMap()),
sg_dg_dxdot->numVectors()));
}
RCP<Epetra_MultiVector> dg_dxdotdot;
if (sg_dg_dxdotdot != NULL) {
sg_dg_dxdotdot->init(0.0);
dg_dxdotdot = rcp(new Epetra_MultiVector(*(sg_dg_dxdotdot->coefficientMap()),
sg_dg_dxdotdot->numVectors()));
}
RCP<Epetra_MultiVector> dg_dp;
if (sg_dg_dp != NULL) {
sg_dg_dp->init(0.0);
dg_dp = rcp(new Epetra_MultiVector(*(sg_dg_dp->coefficientMap()),
sg_dg_dp->numVectors()));
}
// Get quadrature data
const Teuchos::Array<double>& norms = basis->norm_squared();
const Teuchos::Array< Teuchos::Array<double> >& points =
quad->getQuadPoints();
const Teuchos::Array<double>& weights = quad->getQuadWeights();
const Teuchos::Array< Teuchos::Array<double> >& vals =
quad->getBasisAtQuadPoints();
int nqp = points.size();
// Compute sg_g via quadrature
for (int qp=0; qp<nqp; qp++) {
// Evaluate sg_x, sg_xdot at quadrature point
sg_x.evaluate(vals[qp], x);
if (sg_xdot != NULL)
sg_xdot->evaluate(vals[qp], *xdot);
if (sg_xdotdot != NULL)
sg_xdotdot->evaluate(vals[qp], *xdotdot);
// Evaluate parameters at quadrature point
for (int i=0; i<sg_p_index.size(); i++) {
int ii = sg_p_index[i];
for (unsigned int j=0; j<pp[ii].size(); j++) {
pp[ii][j].baseValue = sg_p_vals[ii][j].evaluate(points[qp], vals[qp]);
if (deriv_p != NULL) {
for (unsigned int k=0; k<deriv_p->size(); k++)
if ((*deriv_p)[k].family->getName() == pp[ii][j].family->getName())
(*deriv_p)[k].baseValue = pp[ii][j].baseValue;
}
}
}
// Compute response at quadrature point
evaluateGradient(current_time, xdot.get(), xdotdot.get(), x, pp, deriv_p,
g.get(), dg_dx.get(), dg_dxdot.get(), dg_dxdotdot.get(), dg_dp.get());
// Add result into integral
if (sg_g != NULL)
sg_g->sumIntoAllTerms(weights[qp], vals[qp], norms, *g);
if (sg_dg_dx != NULL)
sg_dg_dx->sumIntoAllTerms(weights[qp], vals[qp], norms, *dg_dx);
if (sg_dg_dxdot != NULL)
sg_dg_dxdot->sumIntoAllTerms(weights[qp], vals[qp], norms, *dg_dxdot);
if (sg_dg_dxdotdot != NULL)
sg_dg_dxdotdot->sumIntoAllTerms(weights[qp], vals[qp], norms, *dg_dxdotdot);
if (sg_dg_dp != NULL)
sg_dg_dp->sumIntoAllTerms(weights[qp], vals[qp], norms, *dg_dp);
}
}
TEUCHOS_UNIT_TEST_TEMPLATE_1_DECL(normals,test2d,EvalType)
{
// build global (or serial communicator)
#ifdef HAVE_MPI
Teuchos::RCP<Epetra_Comm> eComm = Teuchos::rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
#else
Teuchos::RCP<Epetra_Comm> eComm = Teuchos::rcp(new Epetra_SerialComm());
#endif
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcp_dynamic_cast;
// panzer::pauseToAttach();
// build a dummy workset
//////////////////////////////////////////////////////////
// typedef Kokkos::DynRankView<double,PHX::Device> FieldArray;
int numCells = 2, numVerts = 4, dim = 2;
Teuchos::RCP<panzer::Workset> workset = Teuchos::rcp(new panzer::Workset);
// FieldArray & coords = workset->cell_vertex_coordinates;
// coords.resize(numCells,numVerts,dim);
MDFieldArrayFactory af("",true);
workset->cell_vertex_coordinates = af.buildStaticArray<double,Cell,NODE,Dim>("coords",numCells,numVerts,dim);
Workset::CellCoordArray coords = workset->cell_vertex_coordinates;
coords(0,0,0) = 1.0; coords(0,0,1) = 0.0;
coords(0,1,0) = 1.0; coords(0,1,1) = 1.0;
coords(0,2,0) = 0.0; coords(0,2,1) = 1.0;
coords(0,3,0) = 0.0; coords(0,3,1) = 0.0;
coords(1,0,0) = 1.0; coords(1,0,1) = 1.0;
coords(1,1,0) = 2.0; coords(1,1,1) = 2.0;
coords(1,2,0) = 1.0; coords(1,2,1) = 3.0;
coords(1,3,0) = 0.0; coords(1,3,1) = 2.0;
Teuchos::RCP<shards::CellTopology> topo =
Teuchos::rcp(new shards::CellTopology(shards::getCellTopologyData< shards::Quadrilateral<4> >()));
int quadOrder = 5;
panzer::CellData cellData(2,1,topo);
Teuchos::RCP<panzer::IntegrationRule> quadRule = Teuchos::rcp(new panzer::IntegrationRule(quadOrder,cellData));
out << "num quad points = " << quadRule->num_points << std::endl;
Teuchos::RCP<panzer::IntegrationValues2<double> > quadValues = Teuchos::rcp(new panzer::IntegrationValues2<double>("",true));
quadValues->setupArrays(quadRule);
quadValues->evaluateValues(coords);
workset->cell_local_ids.push_back(0); workset->cell_local_ids.push_back(1);
workset->num_cells = numCells;
workset->block_id = "eblock-0_0";
workset->ir_degrees = Teuchos::rcp(new std::vector<int>);
workset->ir_degrees->push_back(quadRule->cubature_degree);
workset->int_rules.push_back(quadValues);
Teuchos::RCP<PHX::FieldManager<panzer::Traits> > fm
= Teuchos::rcp(new PHX::FieldManager<panzer::Traits>);
// typedef panzer::Traits::Residual EvalType;
typedef Sacado::ScalarValue<typename EvalType::ScalarT> ScalarValue;
Teuchos::RCP<PHX::MDField<typename EvalType::ScalarT,panzer::Cell,panzer::Point,panzer::Dim> > normalsPtr;
{
Teuchos::ParameterList p;
p.set("Name","Norms");
p.set("IR",quadRule);
p.set("Side ID",1);
RCP<panzer::Normals<EvalType,panzer::Traits> > normEval
= rcp(new panzer::Normals<EvalType,panzer::Traits>(p));
RCP<PHX::Evaluator<panzer::Traits> > eval = normEval;
fm->registerEvaluator<EvalType>(eval);
fm->requireField<EvalType>(normEval->getFieldTag());
const PHX::FieldTag & ft = normEval->getFieldTag();
normalsPtr = rcp(new
PHX::MDField<typename EvalType::ScalarT,panzer::Cell,panzer::Point,panzer::Dim>(ft.name(),quadRule->dl_vector));
}
PHX::MDField<typename EvalType::ScalarT,panzer::Cell,panzer::Point,panzer::Dim> & normals = *normalsPtr;
panzer::Traits::SetupData setupData;
setupData.worksets_ = rcp(new std::vector<panzer::Workset>);
setupData.worksets_->push_back(*workset);
std::vector<PHX::index_size_type> derivative_dimensions;
derivative_dimensions.push_back(4);
fm->setKokkosExtendedDataTypeDimensions<panzer::Traits::Jacobian>(derivative_dimensions);
#ifdef Panzer_BUILD_HESSIAN_SUPPORT
fm->setKokkosExtendedDataTypeDimensions<panzer::Traits::Hessian>(derivative_dimensions);
#endif
fm->postRegistrationSetup(setupData);
panzer::Traits::PreEvalData preEvalData;
fm->preEvaluate<EvalType>(preEvalData);
fm->evaluateFields<EvalType>(*workset);
fm->postEvaluate<EvalType>(0);
fm->getFieldData<typename EvalType::ScalarT,EvalType>(normals);
TEST_EQUALITY(normals.rank(),3);
TEST_EQUALITY(normals.size(),numCells*quadRule->num_points*dim);
normals.print(out,false);
for(int i=0;i<numCells;i++) {
// useful for checking if normals are consistent: transformation is
// affine!
double nx0 = ScalarValue::eval(normals(i,0,0));
double ny0 = ScalarValue::eval(normals(i,0,1));
for(int v=0;v<quadRule->num_points;v++) {
double nx = ScalarValue::eval(normals(i,v,0));
double ny = ScalarValue::eval(normals(i,v,1));
TEST_FLOATING_EQUALITY(nx*nx+ny*ny,1.0,1e-15);
// check point consistency
TEST_FLOATING_EQUALITY(nx,nx0,1e-15);
TEST_FLOATING_EQUALITY(ny,ny0,1e-15);
}
}
// check cell 0
{
double nx = ScalarValue::eval(normals(0,0,0));
double ny = ScalarValue::eval(normals(0,0,1));
TEST_FLOATING_EQUALITY(nx,0.0,1e-15);
TEST_FLOATING_EQUALITY(ny,1.0,1e-15);
}
// check cell 1
{
double nx = ScalarValue::eval(normals(1,0,0));
double ny = ScalarValue::eval(normals(1,0,1));
double sqrt2 = std::sqrt(2.0);
TEST_FLOATING_EQUALITY(nx,1.0/sqrt2,1e-15);
TEST_FLOATING_EQUALITY(ny,1.0/sqrt2,1e-15);
}
}
int main(int argc, char *argv[])
{
Teuchos::GlobalMPISession session(&argc, &argv);
RCP<const Comm<int> > comm = DefaultComm<int>::getComm();
int rank = comm->getRank();
int fail = 0, gfail=0;
// Create object that can give us test Tpetra, Xpetra
// and Epetra vectors for testing.
RCP<UserInputForTests> uinput;
try{
uinput =
rcp(new UserInputForTests(testDataFilePath,std::string("simple"), comm, true));
}
catch(std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0, string("input ")+e.what(), 1);
}
RCP<tvector_t> tV; // original vector (for checking)
RCP<tvector_t> newV; // migrated vector
int nVec = 2;
tV = rcp(new tvector_t(uinput->getUITpetraCrsGraph()->getRowMap(), nVec));
tV->randomize();
size_t vlen = tV->getLocalLength();
// To test migration in the input adapter we need a Solution object.
RCP<const Zoltan2::Environment> env = rcp(new Zoltan2::Environment);
int nWeights = 1;
typedef Zoltan2::XpetraMultiVectorAdapter<tvector_t> ia_t;
typedef Zoltan2::PartitioningSolution<ia_t> soln_t;
typedef ia_t::part_t part_t;
part_t *p = new part_t [vlen];
memset(p, 0, sizeof(part_t) * vlen);
ArrayRCP<part_t> solnParts(p, 0, vlen, true);
soln_t solution(env, comm, nWeights);
solution.setParts(solnParts);
std::vector<const zscalar_t *> emptyWeights;
std::vector<int> emptyStrides;
/////////////////////////////////////////////////////////////
// User object is Tpetra::MultiVector, no weights
if (!gfail){
RCP<const tvector_t> ctV = rcp_const_cast<const tvector_t>(tV);
RCP<Zoltan2::XpetraMultiVectorAdapter<tvector_t> > tVInput;
try {
tVInput =
rcp(new Zoltan2::XpetraMultiVectorAdapter<tvector_t>(ctV,
emptyWeights, emptyStrides));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraMultiVectorAdapter ")+e.what(), 1);
}
if (rank==0){
std::cout << "Constructed with ";
std::cout << "Tpetra::MultiVector" << std::endl;
}
fail = verifyInputAdapter<tvector_t>(*tVInput, *tV, nVec, 0, NULL, NULL);
gfail = globalFail(comm, fail);
if (!gfail){
tvector_t *vMigrate = NULL;
try{
tVInput->applyPartitioningSolution(*tV, vMigrate, solution);
newV = rcp(vMigrate);
}
catch (std::exception &e){
fail = 11;
}
gfail = globalFail(comm, fail);
if (!gfail){
RCP<const tvector_t> cnewV = rcp_const_cast<const tvector_t>(newV);
RCP<Zoltan2::XpetraMultiVectorAdapter<tvector_t> > newInput;
try{
newInput = rcp(new Zoltan2::XpetraMultiVectorAdapter<tvector_t>(
cnewV, emptyWeights, emptyStrides));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraMultiVectorAdapter 2 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Constructed with ";
std::cout << "Tpetra::MultiVector migrated to proc 0" << std::endl;
}
fail = verifyInputAdapter<tvector_t>(*newInput, *newV, nVec, 0, NULL, NULL);
if (fail) fail += 100;
gfail = globalFail(comm, fail);
}
}
if (gfail){
printFailureCode(comm, fail);
}
}
/////////////////////////////////////////////////////////////
// User object is Xpetra::MultiVector
if (!gfail){
RCP<tvector_t> tMV =
rcp(new tvector_t(uinput->getUITpetraCrsGraph()->getRowMap(), nVec));
tMV->randomize();
RCP<xvector_t> xV = Zoltan2::XpetraTraits<tvector_t>::convertToXpetra(tMV);
RCP<const xvector_t> cxV = rcp_const_cast<const xvector_t>(xV);
RCP<Zoltan2::XpetraMultiVectorAdapter<xvector_t> > xVInput;
try {
xVInput =
rcp(new Zoltan2::XpetraMultiVectorAdapter<xvector_t>(cxV,
emptyWeights, emptyStrides));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraMultiVectorAdapter 3 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Constructed with ";
std::cout << "Xpetra::MultiVector" << std::endl;
}
fail = verifyInputAdapter<xvector_t>(*xVInput, *tV, nVec, 0, NULL, NULL);
gfail = globalFail(comm, fail);
if (!gfail){
xvector_t *vMigrate =NULL;
try{
xVInput->applyPartitioningSolution(*xV, vMigrate, solution);
}
catch (std::exception &e){
fail = 11;
}
gfail = globalFail(comm, fail);
if (!gfail){
RCP<const xvector_t> cnewV(vMigrate);
RCP<Zoltan2::XpetraMultiVectorAdapter<xvector_t> > newInput;
try{
newInput =
rcp(new Zoltan2::XpetraMultiVectorAdapter<xvector_t>(
cnewV, emptyWeights, emptyStrides));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraMultiVectorAdapter 4 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Constructed with ";
std::cout << "Xpetra::MultiVector migrated to proc 0" << std::endl;
}
fail = verifyInputAdapter<xvector_t>(*newInput, *newV, nVec, 0, NULL, NULL);
if (fail) fail += 100;
gfail = globalFail(comm, fail);
}
}
if (gfail){
printFailureCode(comm, fail);
}
}
#ifdef HAVE_EPETRA_DATA_TYPES
/////////////////////////////////////////////////////////////
// User object is Epetra_MultiVector
if (!gfail){
RCP<evector_t> eV =
rcp(new Epetra_MultiVector(uinput->getUIEpetraCrsGraph()->RowMap(),
nVec));
eV->Random();
RCP<const evector_t> ceV = rcp_const_cast<const evector_t>(eV);
RCP<Zoltan2::XpetraMultiVectorAdapter<evector_t> > eVInput;
try {
eVInput =
rcp(new Zoltan2::XpetraMultiVectorAdapter<evector_t>(ceV,
emptyWeights, emptyStrides));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraMultiVectorAdapter 5 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Constructed with ";
std::cout << "Epetra_MultiVector" << std::endl;
}
fail = verifyInputAdapter<evector_t>(*eVInput, *tV, nVec, 0, NULL, NULL);
gfail = globalFail(comm, fail);
if (!gfail){
evector_t *vMigrate =NULL;
try{
eVInput->applyPartitioningSolution(*eV, vMigrate, solution);
}
catch (std::exception &e){
fail = 11;
}
gfail = globalFail(comm, fail);
if (!gfail){
RCP<const evector_t> cnewV(vMigrate, true);
RCP<Zoltan2::XpetraMultiVectorAdapter<evector_t> > newInput;
try{
newInput =
rcp(new Zoltan2::XpetraMultiVectorAdapter<evector_t>(cnewV,
emptyWeights, emptyStrides));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraMultiVectorAdapter 6 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Constructed with ";
std::cout << "Epetra_MultiVector migrated to proc 0" << std::endl;
}
fail = verifyInputAdapter<evector_t>(*newInput, *newV, nVec, 0, NULL, NULL);
if (fail) fail += 100;
gfail = globalFail(comm, fail);
}
}
if (gfail){
printFailureCode(comm, fail);
}
}
#endif
/////////////////////////////////////////////////////////////
// DONE
if (rank==0)
std::cout << "PASS" << std::endl;
}
TopRAPFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::TopRAPFactory(RCP<const FactoryManagerBase> parentFactoryManagerFine, RCP<const FactoryManagerBase> parentFactoryManagerCoarse)
: PFact_(parentFactoryManagerCoarse->GetFactory("P")), RFact_(parentFactoryManagerCoarse->GetFactory("R")), AcFact_(parentFactoryManagerCoarse->GetFactory("A"))
{ }
TEUCHOS_UNIT_TEST(assembly_engine, dirichlet_only)
{
using Teuchos::RCP;
using Teuchos::rcp_dynamic_cast;
RCP<Teuchos::ParameterList> pl = rcp(new Teuchos::ParameterList);
pl->set("X Blocks",2);
pl->set("Y Blocks",1);
pl->set("X Elements",6);
pl->set("Y Elements",4);
panzer_stk_classic::SquareQuadMeshFactory factory;
factory.setParameterList(pl);
RCP<panzer_stk_classic::STK_Interface> mesh = factory.buildMesh(MPI_COMM_WORLD);
RCP<Epetra_Comm> Comm = Teuchos::rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
Teuchos::RCP<Teuchos::ParameterList> ipb = Teuchos::parameterList("Physics Blocks");
std::vector<panzer::BC> bcs;
testInitialzation(ipb, bcs);
Teuchos::RCP<panzer::FieldManagerBuilder> fmb =
Teuchos::rcp(new panzer::FieldManagerBuilder);
// build physics blocks
//////////////////////////////////////////////////////////////
const std::size_t workset_size = 20;
Teuchos::RCP<user_app::MyFactory> eqset_factory = Teuchos::rcp(new user_app::MyFactory);
user_app::BCFactory bc_factory;
std::vector<Teuchos::RCP<panzer::PhysicsBlock> > physicsBlocks;
{
std::map<std::string,std::string> block_ids_to_physics_ids;
block_ids_to_physics_ids["eblock-0_0"] = "test physics";
block_ids_to_physics_ids["eblock-1_0"] = "test physics";
std::map<std::string,Teuchos::RCP<const shards::CellTopology> > block_ids_to_cell_topo;
block_ids_to_cell_topo["eblock-0_0"] = mesh->getCellTopology("eblock-0_0");
block_ids_to_cell_topo["eblock-1_0"] = mesh->getCellTopology("eblock-1_0");
Teuchos::RCP<panzer::GlobalData> gd = panzer::createGlobalData();
int default_integration_order = 1;
panzer::buildPhysicsBlocks(block_ids_to_physics_ids,
block_ids_to_cell_topo,
ipb,
default_integration_order,
workset_size,
eqset_factory,
gd,
false,
physicsBlocks);
}
// build worksets
//////////////////////////////////////////////////////////////
Teuchos::RCP<panzer_stk_classic::WorksetFactory> wkstFactory
= Teuchos::rcp(new panzer_stk_classic::WorksetFactory(mesh)); // build STK workset factory
Teuchos::RCP<panzer::WorksetContainer> wkstContainer // attach it to a workset container (uses lazy evaluation)
= Teuchos::rcp(new panzer::WorksetContainer(wkstFactory,physicsBlocks,workset_size));
// build DOF Manager
/////////////////////////////////////////////////////////////
// build the connection manager
const Teuchos::RCP<panzer::ConnManager<int,int> >
conn_manager = Teuchos::rcp(new panzer_stk_classic::STKConnManager<int>(mesh));
panzer::DOFManagerFactory<int,int> globalIndexerFactory;
RCP<panzer::UniqueGlobalIndexer<int,int> > dofManager
= globalIndexerFactory.buildUniqueGlobalIndexer(Teuchos::opaqueWrapper(MPI_COMM_WORLD),physicsBlocks,conn_manager);
Teuchos::RCP<panzer::EpetraLinearObjFactory<panzer::Traits,int> > eLinObjFactory
= Teuchos::rcp(new panzer::EpetraLinearObjFactory<panzer::Traits,int>(Comm.getConst(),dofManager));
Teuchos::RCP<panzer::LinearObjFactory<panzer::Traits> > linObjFactory = eLinObjFactory;
// setup field manager build
/////////////////////////////////////////////////////////////
// Add in the application specific closure model factory
panzer::ClosureModelFactory_TemplateManager<panzer::Traits> cm_factory;
user_app::MyModelFactory_TemplateBuilder cm_builder;
cm_factory.buildObjects(cm_builder);
Teuchos::ParameterList closure_models("Closure Models");
closure_models.sublist("solid").sublist("SOURCE_TEMPERATURE").set<double>("Value",1.0);
closure_models.sublist("ion solid").sublist("SOURCE_ION_TEMPERATURE").set<double>("Value",1.0);
Teuchos::ParameterList user_data("User Data");
fmb->setWorksetContainer(wkstContainer);
fmb->setupVolumeFieldManagers(physicsBlocks,cm_factory,closure_models,*linObjFactory,user_data);
fmb->setupBCFieldManagers(bcs,physicsBlocks,*eqset_factory,cm_factory,bc_factory,closure_models,*linObjFactory,user_data);
panzer::AssemblyEngine_TemplateManager<panzer::Traits> ae_tm;
panzer::AssemblyEngine_TemplateBuilder builder(fmb,linObjFactory);
ae_tm.buildObjects(builder);
RCP<panzer::LinearObjContainer> ghosted = linObjFactory->buildGhostedLinearObjContainer();
RCP<panzer::LinearObjContainer> global = linObjFactory->buildLinearObjContainer();
eLinObjFactory->initializeGhostedContainer(panzer::EpetraLinearObjContainer::X |
panzer::EpetraLinearObjContainer::DxDt |
panzer::EpetraLinearObjContainer::F,*ghosted);
eLinObjFactory->initializeContainer(panzer::EpetraLinearObjContainer::X |
panzer::EpetraLinearObjContainer::DxDt |
panzer::EpetraLinearObjContainer::F,*global);
ghosted->initialize();
global->initialize();
Thyra::assign(Teuchos::rcp_dynamic_cast<panzer::ThyraObjContainer<double> >(global,true)->get_x_th().ptr(),1.0);
Thyra::assign(Teuchos::rcp_dynamic_cast<panzer::ThyraObjContainer<double> >(ghosted,true)->get_x_th().ptr(),-33.0);
panzer::AssemblyEngineInArgs input(ghosted,global);
input.alpha = 0.0;
input.beta = 1.0;
Teuchos::RCP<panzer::LinearObjContainer> counter = ae_tm.getAsObject<panzer::Traits::Residual>()->evaluateOnlyDirichletBCs(input);
TEST_ASSERT(counter!=Teuchos::null);
out << "evaluated" << std::endl;
RCP<Thyra::VectorBase<double> > count = Teuchos::rcp_dynamic_cast<panzer::ThyraObjContainer<double> >(counter,true)->get_x_th();
RCP<Thyra::VectorBase<double> > values = Teuchos::rcp_dynamic_cast<panzer::ThyraObjContainer<double> >(global,true)->get_f_th();
TEST_ASSERT(count!=Teuchos::null);
TEST_ASSERT(values!=Teuchos::null);
Teuchos::ArrayRCP<double> count_array, values_array;
rcp_dynamic_cast<Thyra::SpmdVectorBase<double> >(count,true)->getNonconstLocalData(Teuchos::ptrFromRef(count_array));
rcp_dynamic_cast<Thyra::SpmdVectorBase<double> >(values,true)->getNonconstLocalData(Teuchos::ptrFromRef(values_array));
bool passed = true;
TEST_EQUALITY(count_array.size(),values_array.size());
for(Teuchos::ArrayRCP<double>::size_type i=0;i<count_array.size();i++) {
if(count_array[i]==0.0)
passed &= (values_array[i]==0.0);
else {
passed &= (std::fabs((values_array[i]-(-4.0))/4.0)<1e-14);
out << values_array[i] << " " << i <<std::endl;
}
}
TEST_ASSERT(passed);
}
void Piro::RythmosSolver<Scalar>::evalModelImpl(
const Thyra::ModelEvaluatorBase::InArgs<Scalar>& inArgs,
const Thyra::ModelEvaluatorBase::OutArgs<Scalar>& outArgs) const
{
using Teuchos::RCP;
using Teuchos::rcp;
// TODO: Support more than 1 parameter and 1 response
const int j = 0;
const int l = 0;
// Parse InArgs
RCP<const Thyra::VectorBase<Scalar> > p_in;
if (num_p > 0) {
p_in = inArgs.get_p(l);
}
// Parse OutArgs
RCP<Thyra::VectorBase<Scalar> > g_out;
if (num_g > 0) {
g_out = outArgs.get_g(j);
}
const RCP<Thyra::VectorBase<Scalar> > gx_out = outArgs.get_g(num_g);
Thyra::ModelEvaluatorBase::InArgs<Scalar> state_ic = model->getNominalValues();
// Set initial time in ME if needed
if(t_initial > 0.0 && state_ic.supports(Thyra::ModelEvaluatorBase::IN_ARG_t))
state_ic.set_t(t_initial);
if (Teuchos::nonnull(initialConditionModel)) {
// The initial condition depends on the parameter
// It is found by querying the auxiliary model evaluator as the last response
const RCP<Thyra::VectorBase<Scalar> > initialState =
Thyra::createMember(model->get_x_space());
{
Thyra::ModelEvaluatorBase::InArgs<Scalar> initCondInArgs = initialConditionModel->createInArgs();
if (num_p > 0) {
initCondInArgs.set_p(l, inArgs.get_p(l));
}
Thyra::ModelEvaluatorBase::OutArgs<Scalar> initCondOutArgs = initialConditionModel->createOutArgs();
initCondOutArgs.set_g(initCondOutArgs.Ng() - 1, initialState);
initialConditionModel->evalModel(initCondInArgs, initCondOutArgs);
}
state_ic.set_x(initialState);
}
// Set paramters p_in as part of initial conditions
if (num_p > 0) {
if (Teuchos::nonnull(p_in)) {
state_ic.set_p(l, p_in);
}
}
*out << "\nstate_ic:\n" << Teuchos::describe(state_ic, solnVerbLevel);
RCP<Thyra::MultiVectorBase<Scalar> > dgxdp_out;
Thyra::ModelEvaluatorBase::Derivative<Scalar> dgdp_deriv_out;
if (num_p > 0) {
const Thyra::ModelEvaluatorBase::DerivativeSupport dgxdp_support =
outArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp, num_g, l);
if (dgxdp_support.supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM)) {
const Thyra::ModelEvaluatorBase::Derivative<Scalar> dgxdp_deriv =
outArgs.get_DgDp(num_g, l);
dgxdp_out = dgxdp_deriv.getMultiVector();
}
if (num_g > 0) {
const Thyra::ModelEvaluatorBase::DerivativeSupport dgdp_support =
outArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp, j, l);
if (!dgdp_support.none()) {
dgdp_deriv_out = outArgs.get_DgDp(j, l);
}
}
}
const bool requestedSensitivities =
Teuchos::nonnull(dgxdp_out) || !dgdp_deriv_out.isEmpty();
RCP<const Thyra::VectorBase<Scalar> > finalSolution;
if (!requestedSensitivities) {
//
*out << "\nE) Solve the forward problem ...\n";
//
fwdStateStepper->setInitialCondition(state_ic);
fwdStateIntegrator->setStepper(fwdStateStepper, t_final, true);
Teuchos::Array<RCP<const Thyra::VectorBase<Scalar> > > x_final_array;
fwdStateIntegrator->getFwdPoints(
Teuchos::tuple<Scalar>(t_final), &x_final_array, NULL, NULL);
finalSolution = x_final_array[0];
if (Teuchos::VERB_MEDIUM <= solnVerbLevel) {
std::cout << "Final Solution\n" << *finalSolution << std::endl;
}
} else { // Computing sensitivities
//
*out << "\nE) Solve the forward problem with Sensitivities...\n";
//
RCP<Rythmos::ForwardSensitivityStepper<Scalar> > stateAndSensStepper =
Rythmos::forwardSensitivityStepper<Scalar>();
stateAndSensStepper->initializeSyncedSteppers(
model, l, model->getNominalValues(),
fwdStateStepper, fwdTimeStepSolver);
//
// Set the initial condition for the state and forward sensitivities
//
const RCP<Thyra::VectorBase<Scalar> > s_bar_init =
Thyra::createMember(stateAndSensStepper->getFwdSensModel()->get_x_space());
const RCP<Thyra::VectorBase<Scalar> > s_bar_dot_init =
Thyra::createMember(stateAndSensStepper->getFwdSensModel()->get_x_space());
if (Teuchos::is_null(initialConditionModel)) {
// The initial condition is assumed to be independent from the parameters
// Therefore, the initial condition for the sensitivity is zero
Thyra::assign(s_bar_init.ptr(), Teuchos::ScalarTraits<Scalar>::zero());
} else {
// Use initialConditionModel to compute initial condition for sensitivity
Thyra::ModelEvaluatorBase::InArgs<Scalar> initCondInArgs = initialConditionModel->createInArgs();
initCondInArgs.set_p(l, inArgs.get_p(l));
Thyra::ModelEvaluatorBase::OutArgs<Scalar> initCondOutArgs = initialConditionModel->createOutArgs();
typedef Thyra::DefaultMultiVectorProductVector<Scalar> DMVPV;
const RCP<DMVPV> s_bar_init_downcasted = Teuchos::rcp_dynamic_cast<DMVPV>(s_bar_init);
const Thyra::ModelEvaluatorBase::Derivative<Scalar> initCond_deriv(
s_bar_init_downcasted->getNonconstMultiVector(),
Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM);
initCondOutArgs.set_DgDp(initCondOutArgs.Ng() - 1, l, initCond_deriv);
initialConditionModel->evalModel(initCondInArgs, initCondOutArgs);
}
Thyra::assign(s_bar_dot_init.ptr(), Teuchos::ScalarTraits<Scalar>::zero());
RCP<const Rythmos::StateAndForwardSensitivityModelEvaluator<Scalar> >
stateAndSensModel = stateAndSensStepper->getStateAndFwdSensModel();
Thyra::ModelEvaluatorBase::InArgs<Scalar>
state_and_sens_ic = stateAndSensStepper->getModel()->createInArgs();
// Copy time, parameters etc.
state_and_sens_ic.setArgs(state_ic);
// Set initial condition for x_bar = [ x; s_bar ]
state_and_sens_ic.set_x(stateAndSensModel->create_x_bar_vec(state_ic.get_x(), s_bar_init));
// Set initial condition for x_bar_dot = [ x_dot; s_bar_dot ]
state_and_sens_ic.set_x_dot(stateAndSensModel->create_x_bar_vec(state_ic.get_x_dot(), s_bar_dot_init));
stateAndSensStepper->setInitialCondition(state_and_sens_ic);
//
// Use a StepperAsModelEvaluator to integrate the state+sens
//
const RCP<Rythmos::StepperAsModelEvaluator<Scalar> >
stateAndSensIntegratorAsModel = Rythmos::stepperAsModelEvaluator(
Teuchos::rcp_implicit_cast<Rythmos::StepperBase<Scalar> >(stateAndSensStepper),
Teuchos::rcp_implicit_cast<Rythmos::IntegratorBase<Scalar> >(fwdStateIntegrator),
state_and_sens_ic);
// StepperAsModelEvaluator outputs the solution as its last response
const int stateAndSensModelStateResponseIndex = stateAndSensIntegratorAsModel->Ng() - 1;
*out << "\nUse the StepperAsModelEvaluator to integrate state + sens x_bar(p,t_final) ... \n";
Teuchos::OSTab tab(out);
// Solution sensitivity in column-oriented (Jacobian) MultiVector form
RCP<const Thyra::MultiVectorBase<Scalar> > dxdp;
const RCP<Thyra::VectorBase<Scalar> > x_bar_final =
Thyra::createMember(stateAndSensIntegratorAsModel->get_g_space(stateAndSensModelStateResponseIndex));
// Extract pieces of x_bar_final to prepare output
{
const RCP<const Thyra::ProductVectorBase<Scalar> > x_bar_final_downcasted =
Thyra::productVectorBase<Scalar>(x_bar_final);
// Solution
const int solutionBlockIndex = 0;
finalSolution = x_bar_final_downcasted->getVectorBlock(solutionBlockIndex);
// Sensitivity
const int sensitivityBlockIndex = 1;
const RCP<const Thyra::VectorBase<Scalar> > s_bar_final =
x_bar_final_downcasted->getVectorBlock(sensitivityBlockIndex);
{
typedef Thyra::DefaultMultiVectorProductVector<Scalar> DMVPV;
const RCP<const DMVPV> s_bar_final_downcasted = Teuchos::rcp_dynamic_cast<const DMVPV>(s_bar_final);
dxdp = s_bar_final_downcasted->getMultiVector();
}
}
Thyra::eval_g(
*stateAndSensIntegratorAsModel,
l, *state_ic.get_p(l),
t_final,
stateAndSensModelStateResponseIndex, x_bar_final.get()
);
*out
<< "\nx_bar_final = x_bar(p,t_final) evaluated using "
<< "stateAndSensIntegratorAsModel:\n"
<< Teuchos::describe(*x_bar_final,solnVerbLevel);
if (Teuchos::nonnull(dgxdp_out)) {
Thyra::assign(dgxdp_out.ptr(), *dxdp);
}
if (!dgdp_deriv_out.isEmpty()) {
RCP<Thyra::DefaultAddedLinearOp<Scalar> > dgdp_op_out;
{
const RCP<Thyra::LinearOpBase<Scalar> > dgdp_op = dgdp_deriv_out.getLinearOp();
if (Teuchos::nonnull(dgdp_op)) {
dgdp_op_out = Teuchos::rcp_dynamic_cast<Thyra::DefaultAddedLinearOp<Scalar> >(dgdp_op);
dgdp_op_out.assert_not_null();
}
}
Thyra::ModelEvaluatorBase::InArgs<Scalar> modelInArgs = model->createInArgs();
{
modelInArgs.set_x(finalSolution);
if (num_p > 0) {
modelInArgs.set_p(l, p_in);
}
}
// require dgdx, dgdp from model
Thyra::ModelEvaluatorBase::OutArgs<Scalar> modelOutArgs = model->createOutArgs();
{
const Thyra::ModelEvaluatorBase::Derivative<Scalar> dgdx_deriv(model->create_DgDx_op(j));
modelOutArgs.set_DgDx(j, dgdx_deriv);
Thyra::ModelEvaluatorBase::Derivative<Scalar> dgdp_deriv;
if (Teuchos::nonnull(dgdp_op_out)) {
dgdp_deriv = model->create_DgDp_op(j, l);
} else {
dgdp_deriv = dgdp_deriv_out;
}
modelOutArgs.set_DgDp(j, l, dgdp_deriv);
}
model->evalModel(modelInArgs, modelOutArgs);
const RCP<const Thyra::LinearOpBase<Scalar> > dgdx =
modelOutArgs.get_DgDx(j).getLinearOp();
// dgdp_out = dgdp + <dgdx, dxdp>
if (Teuchos::nonnull(dgdp_op_out)) {
Teuchos::Array<RCP<const Thyra::LinearOpBase<Scalar> > > op_args(2);
{
op_args[0] = modelOutArgs.get_DgDp(j, l).getLinearOp();
op_args[1] = Thyra::multiply<Scalar>(dgdx, dxdp);
}
dgdp_op_out->initialize(op_args);
} else {
const RCP<Thyra::MultiVectorBase<Scalar> > dgdp_mv_out = dgdp_deriv_out.getMultiVector();
Thyra::apply(
*dgdx,
Thyra::NOTRANS,
*dxdp,
dgdp_mv_out.ptr(),
Teuchos::ScalarTraits<Scalar>::one(),
Teuchos::ScalarTraits<Scalar>::one());
}
}
}
*out << "\nF) Check the solution to the forward problem ...\n";
// As post-processing step, calculate responses at final solution
{
Thyra::ModelEvaluatorBase::InArgs<Scalar> modelInArgs = model->createInArgs();
{
modelInArgs.set_x(finalSolution);
if (num_p > 0) {
modelInArgs.set_p(l, p_in);
}
}
Thyra::ModelEvaluatorBase::OutArgs<Scalar> modelOutArgs = model->createOutArgs();
if (Teuchos::nonnull(g_out)) {
Thyra::put_scalar(Teuchos::ScalarTraits<Scalar>::zero(), g_out.ptr());
modelOutArgs.set_g(j, g_out);
}
model->evalModel(modelInArgs, modelOutArgs);
}
// Return the final solution as an additional g-vector, if requested
if (Teuchos::nonnull(gx_out)) {
Thyra::copy(*finalSolution, gx_out.ptr());
}
}
bool tNeumannSeries::test_simpleOp(int verbosity,std::ostream & os)
{
bool status = false;
bool allPassed = true;
// perform actual test
Thyra::LinearOpTester<double> tester;
tester.show_all_tests(true);
std::stringstream ss;
Teuchos::FancyOStream fos(rcpFromRef(ss)," |||");
{
// build library parameter list
Teuchos::RCP<Teuchos::ParameterList> pl = buildLibPL(1,"None");
if(verbosity>=10) {
os << " tNeumannSeries::test_simpleOp :"
<< " printing library parameter list" << std::endl;
pl->print(os);
}
RCP<Teko::InverseLibrary> invLib = Teko::InverseLibrary::buildFromParameterList(*pl);
RCP<Teko::InverseFactory> neumann = invLib->getInverseFactory("Neumann");
RCP<Teko::InverseFactory> direct = invLib->getInverseFactory("Amesos");
Teko::LinearOp op = buildExampleOp(1,*GetComm());
Teko::LinearOp neuInv = Teko::buildInverse(*neumann,op);
Teko::LinearOp dirInv = Teko::buildInverse(*direct,op);
const bool result = tester.compare( *neuInv, *dirInv, Teuchos::ptrFromRef(fos) );
TEST_ASSERT(not result,
std::endl << " tNeumannSeries::test_simpleOp "
<< ": Comparing underresolved factory generated operator to correct operator");
if(result || verbosity>=10)
os << ss.str();
}
{
// build library parameter list
Teuchos::RCP<Teuchos::ParameterList> pl = buildLibPL(2,"None");
if(verbosity>=10) {
os << " tNeumannSeries::test_simpleOp :"
<< " printing library parameter list" << std::endl;
pl->print(os);
}
RCP<Teko::InverseLibrary> invLib = Teko::InverseLibrary::buildFromParameterList(*pl);
RCP<Teko::InverseFactory> neumann = invLib->getInverseFactory("Neumann");
RCP<Teko::InverseFactory> direct = invLib->getInverseFactory("Amesos");
Teko::LinearOp op = buildExampleOp(1,*GetComm());
Teko::LinearOp neuInv = Teko::buildInverse(*neumann,op);
Teko::LinearOp dirInv = Teko::buildInverse(*direct,op);
const bool result = tester.compare( *neuInv, *dirInv, Teuchos::ptrFromRef(fos) );
TEST_ASSERT(result,
std::endl << " tNeumannSeries::test_simpleOp "
<< ": Comparing factory generated operator to correct operator");
if(not result || verbosity>=10)
os << ss.str();
}
return allPassed;
}
void Example::BCStrategy_Interface_WeakDirichletMatch<EvalT>::
buildAndRegisterEvaluators(PHX::FieldManager<panzer::Traits>& fm,
const panzer::PhysicsBlock& pb,
const panzer::ClosureModelFactory_TemplateManager<panzer::Traits>& factory,
const Teuchos::ParameterList& models,
const Teuchos::ParameterList& user_data) const
{
using Teuchos::ParameterList;
using Teuchos::RCP;
using Teuchos::rcp;
using std::string;
const std::vector<boost::tuples::tuple<string,string,string,int,Teuchos::RCP<panzer::PureBasis>,
Teuchos::RCP<panzer::IntegrationRule> > > data = this->getResidualContributionData();
string residual_name = data[0].get<0>();
string dof_name = data[0].get<1>();
string diff_name = data[0].get<2>();
RCP<panzer::IntegrationRule> ir = data[0].get<5>();
RCP<const panzer::FieldLayoutLibrary> fll = pb.getFieldLibrary()->buildFieldLayoutLibrary(*ir);
RCP<panzer::BasisIRLayout> basis = fll->lookupLayout(dof_name);
if (this->getDetailsIndex() == 0) {
const std::string
dof_grad_name = dof_name + "_gradient",
cancel_natural_name = dof_name + "_cancel",
my_normal_name = "My_Normal",
sum_contributions_name = "Sum_Contributions";
// Weak Dirichlet match.
{
{ // Get values on my side.
ParameterList p("My DOF");
p.set("Name", dof_name);
p.set("Basis", basis);
p.set("IR", ir);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::DOF<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
{ // Other DOF - my DOF.
ParameterList p("other DOF - my DOF");
p.set("Sum Name", diff_name);
setSumValues(p, other_dof_name, 1, dof_name, -1);
p.set("Data Layout", ir->dl_scalar);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::Sum<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
}
// Cancel my natural (Neumann) BC.
{
{ // Normal.
ParameterList p("My Side Normal");
p.set("Name", my_normal_name);
p.set("Side ID", pb.cellData().side());
p.set("IR", ir);
p.set("Normalize", true);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::Normals<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
{ // Gradient.
ParameterList p("My DOF gradient");
p.set("Name", dof_name);
p.set("Gradient Name", dof_grad_name);
p.set("Basis", basis);
p.set("IR", ir);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::DOFGradient<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
{ // dot(DOF gradient, normal).
ParameterList p("dot(my DOF gradient, my normal)");
p.set("Result Name", cancel_natural_name);
p.set("Vector A Name", dof_grad_name);
p.set("Vector B Name", my_normal_name);
p.set("Point Rule", Teuchos::rcp_dynamic_cast<const panzer::PointRule>(ir));
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::DotProduct<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
}
// Add contributions to the residual.
{
{ // Weak Dirichlet Match + Cancel Neumann
ParameterList p("Weak Dirichlet Match + Cancel Neumann");
p.set("Sum Name", sum_contributions_name);
setSumValues(p, diff_name, 1e5, cancel_natural_name, -1);
p.set("Data Layout", ir->dl_scalar);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::Sum<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
{
ParameterList p("Weak Dirichlet Match And Cancel Neumann Residual");
p.set("Residual Name", residual_name);
p.set("Value Name", sum_contributions_name);
p.set("Basis", basis);
p.set("IR", ir);
p.set("Multiplier", 1.0);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::Integrator_BasisTimesScalar<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
}
} else {
{ // Get values on other side.
ParameterList p("Other DOF");
p.set("Name", dof_name);
p.set("Basis", basis);
p.set("IR", ir);
const RCP< PHX::Evaluator<panzer::Traits> >
op = rcp(new panzer::DOF<EvalT,panzer::Traits>(p));
this->template registerEvaluator<EvalT>(fm, op);
}
}
}
TEUCHOS_UNIT_TEST(tEpetra_GlbEvalData, filtered_dofs)
{
using Teuchos::RCP;
using Teuchos::rcp;
typedef Thyra::VectorBase<double> Thyra_Vector;
typedef Thyra::SpmdVectorBase<double> Thyra_SpmdVec;
typedef Thyra::SpmdVectorSpaceBase<double> Thyra_SpmdVecSpace;
Epetra_MpiComm comm(MPI_COMM_WORLD);
// This is required
TEST_ASSERT(comm.NumProc()==2);
std::vector<int> ghosted(4);
std::vector<int> unique(2);
// This is a line with 6 notes (numbered 0-5). The boundaries
// are removed at 0 and 5.
if(comm.MyPID()==0) {
unique[0] = 1;
unique[1] = 2;
ghosted[0] = 0;
ghosted[1] = 1;
ghosted[2] = 2;
ghosted[3] = 3;
}
else {
unique[0] = 3;
unique[1] = 4;
ghosted[0] = 2;
ghosted[1] = 3;
ghosted[2] = 4;
ghosted[3] = 5;
}
RCP<const Epetra_Map> uniqueMap = rcp(new Epetra_Map(-1,unique.size(),&unique[0],0,comm));
RCP<const Epetra_Map> ghostedMap = rcp(new Epetra_Map(-1,ghosted.size(),&ghosted[0],0,comm));
RCP<const Epetra_Import> importer = rcp(new Epetra_Import(*ghostedMap,*uniqueMap));
EpetraVector_ReadOnly_GlobalEvaluationData ged;
std::vector<int> constIndex(1);
// setup filtered values
constIndex[0] = 0;
ged.useConstantValues(constIndex,2.0);
constIndex[0] = 5;
ged.useConstantValues(constIndex,3.0);
ged.initialize(importer,ghostedMap,uniqueMap);
TEST_THROW(ged.useConstantValues(constIndex,4.0),std::logic_error);
{
RCP<Epetra_Vector> uniqueVec = rcp(new Epetra_Vector(*uniqueMap));
if(comm.MyPID()==0) {
(*uniqueVec)[0] = 3.14;
(*uniqueVec)[1] = 1.82;
}
else {
(*uniqueVec)[0] = 2.72;
(*uniqueVec)[1] = 6.23;
}
// set the unique vector, assure that const can be used
ged.setUniqueVector_Epetra(uniqueVec.getConst());
}
// actually do something...
ged.initializeData();
ged.globalToGhost(0);
// check values making sure that the constants are there
{
const Epetra_Vector & ghostedVecE = *ged.getGhostedVector_Epetra();
if(comm.MyPID()==0) {
TEST_EQUALITY(ghostedVecE[0],2.0); // <= Replaced constant value
TEST_EQUALITY(ghostedVecE[1],3.14);
TEST_EQUALITY(ghostedVecE[2],1.82);
TEST_EQUALITY(ghostedVecE[3],2.72);
}
else {
TEST_EQUALITY(ghostedVecE[0],1.82);
TEST_EQUALITY(ghostedVecE[1],2.72);
TEST_EQUALITY(ghostedVecE[2],6.23);
TEST_EQUALITY(ghostedVecE[3],3.0); // <= Replaced constant value
}
}
}
TEUCHOS_UNIT_TEST(tEpetra_GlbEvalData, basic)
{
using Teuchos::RCP;
using Teuchos::rcp;
typedef Thyra::VectorBase<double> Thyra_Vector;
typedef Thyra::SpmdVectorBase<double> Thyra_SpmdVec;
typedef Thyra::SpmdVectorSpaceBase<double> Thyra_SpmdVecSpace;
Epetra_MpiComm comm(MPI_COMM_WORLD);
// This is required
TEST_ASSERT(comm.NumProc()==2);
std::vector<int> ghosted(5);
std::vector<int> unique(3);
if(comm.MyPID()==0) {
unique[0] = 0;
unique[1] = 1;
unique[2] = 2;
ghosted[0] = 0;
ghosted[1] = 1;
ghosted[2] = 2;
ghosted[3] = 3;
ghosted[4] = 4;
}
else {
unique[0] = 3;
unique[1] = 4;
unique[2] = 5;
ghosted[0] = 1;
ghosted[1] = 2;
ghosted[2] = 3;
ghosted[3] = 4;
ghosted[4] = 5;
}
RCP<const Epetra_Map> uniqueMap = rcp(new Epetra_Map(-1,unique.size(),&unique[0],0,comm));
RCP<const Epetra_Map> ghostedMap = rcp(new Epetra_Map(-1,ghosted.size(),&ghosted[0],0,comm));
RCP<const Epetra_Import> importer = rcp(new Epetra_Import(*ghostedMap,*uniqueMap));
EpetraVector_ReadOnly_GlobalEvaluationData ged;
TEST_ASSERT(!ged.isInitialized());
ged.initialize(importer,ghostedMap,uniqueMap);
TEST_ASSERT(ged.isInitialized());
// test the ghosted vector sizing (we don't care what the entries are!)
{
RCP<Epetra_Vector> ghostedVecE = ged.getGhostedVector_Epetra();
RCP<Thyra_Vector> ghostedVecT = ged.getGhostedVector();
TEST_ASSERT(ghostedVecE!=Teuchos::null);
TEST_ASSERT(ghostedVecT!=Teuchos::null);
RCP<const Thyra::SpmdVectorSpaceBase<double> > ghostedSpace
= rcp_dynamic_cast<const Thyra::SpmdVectorSpaceBase<double> >(ghostedVecT->space());
TEST_EQUALITY(ghostedMap->NumMyElements(),ghostedVecE->MyLength());
TEST_EQUALITY(ghostedMap->NumGlobalElements(),ghostedVecE->GlobalLength());
TEST_EQUALITY(ghostedSpace->isLocallyReplicated(),false);
TEST_EQUALITY(ghostedSpace->localSubDim(),ghostedVecE->MyLength());
}
// test setting a unique vector
{
RCP<Epetra_Vector> uniqueVec = rcp(new Epetra_Vector(*uniqueMap));
if(comm.MyPID()==0) {
(*uniqueVec)[0] = 3.14;
(*uniqueVec)[1] = 1.82;
(*uniqueVec)[2] = -.91;
}
else {
(*uniqueVec)[0] = 2.72;
(*uniqueVec)[1] = 6.23;
(*uniqueVec)[2] = -.17;
}
// set the unique vector, assure that const can be used
ged.setUniqueVector_Epetra(uniqueVec.getConst());
}
// test the unique vector sizing and thyra entries
{
RCP<const Thyra_Vector> uniqueVecT = ged.getUniqueVector();
TEST_ASSERT(uniqueVecT!=Teuchos::null);
RCP<const Thyra::SpmdVectorSpaceBase<double> > uniqueSpace
= rcp_dynamic_cast<const Thyra::SpmdVectorSpaceBase<double> >(uniqueVecT->space());
TEST_EQUALITY(uniqueSpace->isLocallyReplicated(),false);
RCP<const Thyra_SpmdVec> spmdVec = rcp_dynamic_cast<const Thyra_SpmdVec>(uniqueVecT);
Teuchos::ArrayRCP<const double> thyraVec;
spmdVec->getLocalData(Teuchos::ptrFromRef(thyraVec));
TEST_EQUALITY(thyraVec.size(),3);
if(comm.MyPID()==0) {
TEST_EQUALITY(thyraVec[0],3.14);
TEST_EQUALITY(thyraVec[1],1.82);
TEST_EQUALITY(thyraVec[2],-.91);
}
else {
TEST_EQUALITY(thyraVec[0],2.72);
TEST_EQUALITY(thyraVec[1],6.23);
TEST_EQUALITY(thyraVec[2],-.17);
}
}
// actually do something...
ged.initializeData();
ged.globalToGhost(0);
{
const Epetra_Vector & ghostedVecE = *ged.getGhostedVector_Epetra();
RCP<Thyra_Vector> ghostedVecT = ged.getGhostedVector();
RCP<const Thyra_SpmdVec> spmdVec = rcp_dynamic_cast<const Thyra_SpmdVec>(ghostedVecT);
Teuchos::ArrayRCP<const double> thyraVec;
spmdVec->getLocalData(Teuchos::ptrFromRef(thyraVec));
TEST_EQUALITY(thyraVec.size(),ghostedVecE.MyLength());
if(comm.MyPID()==0) {
TEST_EQUALITY(ghostedVecE[0],3.14);
TEST_EQUALITY(ghostedVecE[1],1.82);
TEST_EQUALITY(ghostedVecE[2],-.91);
TEST_EQUALITY(ghostedVecE[3],2.72);
TEST_EQUALITY(ghostedVecE[4],6.23);
}
else {
TEST_EQUALITY(ghostedVecE[0],1.82);
TEST_EQUALITY(ghostedVecE[1],-.91);
TEST_EQUALITY(ghostedVecE[2],2.72);
TEST_EQUALITY(ghostedVecE[3],6.23);
TEST_EQUALITY(ghostedVecE[4],-.17);
}
TEST_EQUALITY(ghostedVecE[0],thyraVec[0]);
TEST_EQUALITY(ghostedVecE[1],thyraVec[1]);
TEST_EQUALITY(ghostedVecE[2],thyraVec[2]);
TEST_EQUALITY(ghostedVecE[3],thyraVec[3]);
TEST_EQUALITY(ghostedVecE[4],thyraVec[4]);
}
}
TEUCHOS_UNIT_TEST(tEpetra_GlbEvalData, blocked)
{
using Teuchos::RCP;
using Teuchos::rcp;
typedef Thyra::VectorBase<double> Thyra_Vector;
typedef Thyra::SpmdVectorBase<double> Thyra_SpmdVec;
typedef Thyra::SpmdVectorSpaceBase<double> Thyra_SpmdVecSpace;
Teuchos::RCP<Teuchos::MpiComm<Thyra::Ordinal> > tComm = Teuchos::rcp(new Teuchos::MpiComm<Thyra::Ordinal>(MPI_COMM_WORLD));
Epetra_MpiComm comm(MPI_COMM_WORLD);
// This is required
TEST_ASSERT(comm.NumProc()==2);
std::vector<int> ghosted(5);
std::vector<int> unique(3);
if(comm.MyPID()==0) {
unique[0] = 0;
unique[1] = 1;
unique[2] = 2;
ghosted[0] = 0;
ghosted[1] = 1;
ghosted[2] = 2;
ghosted[3] = 3;
ghosted[4] = 4;
}
else {
unique[0] = 3;
unique[1] = 4;
unique[2] = 5;
ghosted[0] = 1;
ghosted[1] = 2;
ghosted[2] = 3;
ghosted[3] = 4;
ghosted[4] = 5;
}
RCP<const Epetra_Map> uniqueMap = rcp(new Epetra_Map(-1,unique.size(),&unique[0],0,comm));
RCP<const Epetra_Map> ghostedMap = rcp(new Epetra_Map(-1,ghosted.size(),&ghosted[0],0,comm));
RCP<const Epetra_Import> importer = rcp(new Epetra_Import(*ghostedMap,*uniqueMap));
RCP<EpetraVector_ReadOnly_GlobalEvaluationData> ged_a, ged_b;
ged_a = rcp(new EpetraVector_ReadOnly_GlobalEvaluationData(importer,ghostedMap,uniqueMap));
ged_b = rcp(new EpetraVector_ReadOnly_GlobalEvaluationData(importer,ghostedMap,uniqueMap));
std::vector<RCP<ReadOnlyVector_GlobalEvaluationData> > gedBlocks;
gedBlocks.push_back(ged_a);
gedBlocks.push_back(ged_b);
RCP<const Thyra::VectorSpaceBase<double> > uniqueSpace_ab = Thyra::defaultSpmdVectorSpace<double>(tComm,3,-1);
RCP<const Thyra::VectorSpaceBase<double> > ghostedSpace_ab = ged_a->getGhostedVector()->space();
RCP<Thyra::DefaultProductVectorSpace<double> > uniqueSpace = Thyra::productVectorSpace<double>(uniqueSpace_ab,2);
RCP<Thyra::DefaultProductVectorSpace<double> > ghostedSpace = Thyra::productVectorSpace<double>(ghostedSpace_ab,2);
BlockedVector_ReadOnly_GlobalEvaluationData ged;
ged.initialize(ghostedSpace,uniqueSpace,gedBlocks);
RCP<Thyra::VectorBase<double> > uniqueVec = Thyra::createMember(*uniqueSpace);
{
RCP<Thyra::ProductVectorBase<double> > vec = Thyra::castOrCreateNonconstProductVectorBase(uniqueVec);
TEST_ASSERT(vec->productSpace()->numBlocks()==2);
if(comm.MyPID()==0) {
Teuchos::ArrayRCP<double> thyraVec;
rcp_dynamic_cast<Thyra_SpmdVec>(vec->getNonconstVectorBlock(0))->getNonconstLocalData(Teuchos::ptrFromRef(thyraVec));
thyraVec[0] = 3.14;
thyraVec[1] = 1.82;
thyraVec[2] = -.91;
rcp_dynamic_cast<Thyra_SpmdVec>(vec->getNonconstVectorBlock(1))->getNonconstLocalData(Teuchos::ptrFromRef(thyraVec));
thyraVec[0] = 3.14+9.0;
thyraVec[1] = 1.82+9.0;
thyraVec[2] = -.91+9.0;
}
else {
Teuchos::ArrayRCP<double> thyraVec;
rcp_dynamic_cast<Thyra_SpmdVec>(vec->getNonconstVectorBlock(0))->getNonconstLocalData(Teuchos::ptrFromRef(thyraVec));
thyraVec[0] = 2.72;
thyraVec[1] = 6.23;
thyraVec[2] = -.17;
rcp_dynamic_cast<Thyra_SpmdVec>(vec->getNonconstVectorBlock(1))->getNonconstLocalData(Teuchos::ptrFromRef(thyraVec));
thyraVec[0] = 2.72+7.0;
thyraVec[1] = 6.23+7.0;
thyraVec[2] = -.17+7.0;
}
}
ged.setUniqueVector(uniqueVec);
ged.initializeData();
ged.globalToGhost(0);
{
RCP<Thyra::ProductVectorBase<double> > ghostedVec = Thyra::castOrCreateNonconstProductVectorBase(ged.getGhostedVector());
if(comm.MyPID()==0) {
Teuchos::ArrayRCP<const double> thyraVec;
rcp_dynamic_cast<const Thyra_SpmdVec>(ghostedVec->getVectorBlock(0))->getLocalData(Teuchos::ptrFromRef(thyraVec));
TEST_EQUALITY(thyraVec[0],3.14);
TEST_EQUALITY(thyraVec[1],1.82);
TEST_EQUALITY(thyraVec[2],-.91);
TEST_EQUALITY(thyraVec[3],2.72);
TEST_EQUALITY(thyraVec[4],6.23);
rcp_dynamic_cast<const Thyra_SpmdVec>(ghostedVec->getVectorBlock(1))->getLocalData(Teuchos::ptrFromRef(thyraVec));
TEST_EQUALITY(thyraVec[0],3.14+9.0);
TEST_EQUALITY(thyraVec[1],1.82+9.0);
TEST_EQUALITY(thyraVec[2],-.91+9.0);
TEST_EQUALITY(thyraVec[3],2.72+7.0);
TEST_EQUALITY(thyraVec[4],6.23+7.0);
}
else {
Teuchos::ArrayRCP<const double> thyraVec;
rcp_dynamic_cast<const Thyra_SpmdVec>(ghostedVec->getVectorBlock(0))->getLocalData(Teuchos::ptrFromRef(thyraVec));
TEST_EQUALITY(thyraVec[0],1.82);
TEST_EQUALITY(thyraVec[1],-.91);
TEST_EQUALITY(thyraVec[2],2.72);
TEST_EQUALITY(thyraVec[3],6.23);
TEST_EQUALITY(thyraVec[4],-.17);
rcp_dynamic_cast<const Thyra_SpmdVec>(ghostedVec->getVectorBlock(1))->getLocalData(Teuchos::ptrFromRef(thyraVec));
TEST_EQUALITY(thyraVec[0],1.82+9.0);
TEST_EQUALITY(thyraVec[1],-.91+9.0);
TEST_EQUALITY(thyraVec[2],2.72+7.0);
TEST_EQUALITY(thyraVec[3],6.23+7.0);
TEST_EQUALITY(thyraVec[4],-.17+7.0);
}
}
}
int main(int argc, char *argv[])
{
using std::cout;
using std::endl;
typedef std::complex<double> ST;
typedef ScalarTraits<ST> SCT;
typedef SCT::magnitudeType MT;
typedef MultiVector<ST,int> MV;
typedef Operator<ST,int> OP;
typedef Anasazi::MultiVecTraits<ST,MV> MVT;
typedef Anasazi::OperatorTraits<ST,MV,OP> OPT;
ST ONE = SCT::one();
GlobalMPISession mpisess(&argc,&argv,&std::cout);
int info = 0;
int MyPID = 0;
RCP<const Teuchos::Comm<int> > comm =
Tpetra::DefaultPlatform::getDefaultPlatform ().getComm ();
MyPID = rank(*comm);
bool testFailed;
bool verbose = false;
bool debug = false;
bool skinny = true;
std::string filename("mhd1280b.cua");
std::string which("LR");
int nev = 4;
int blockSize = 4;
MT tol = 1.0e-6;
CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("skinny","hefty",&skinny,"Use a skinny (low-mem) or hefty (higher-mem) implementation of IRTR.");
cmdp.setOption("debug","nodebug",&debug,"Print debugging information.");
cmdp.setOption("filename",&filename,"Filename for Harwell-Boeing test matrix.");
cmdp.setOption("sort",&which,"Targetted eigenvalues (SR or LR).");
cmdp.setOption("nev",&nev,"Number of eigenvalues to compute.");
cmdp.setOption("blockSize",&blockSize,"Block size for the algorithm.");
cmdp.setOption("tol",&tol,"Tolerance for convergence.");
if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
return -1;
}
if (debug) verbose = true;
if (blockSize < nev) {
blockSize = nev;
}
if (MyPID == 0) {
cout << Anasazi::Anasazi_Version() << endl << endl;
}
// Get the data from the HB file
int dim,dim2,nnz;
int rnnzmax;
double *dvals;
int *colptr,*rowind;
nnz = -1;
if (MyPID == 0) {
info = readHB_newmat_double(filename.c_str(),&dim,&dim2,&nnz,&colptr,&rowind,&dvals);
// find maximum NNZ over all rows
vector<int> rnnz(dim,0);
for (int *ri=rowind; ri<rowind+nnz; ++ri) {
++rnnz[*ri-1];
}
rnnzmax = *std::max_element(rnnz.begin(),rnnz.end());
}
else {
// address uninitialized data warnings
dvals = NULL;
colptr = NULL;
rowind = NULL;
}
Teuchos::broadcast(*comm,0,&info);
Teuchos::broadcast(*comm,0,&nnz);
Teuchos::broadcast(*comm,0,&dim);
Teuchos::broadcast(*comm,0,&rnnzmax);
if (info == 0 || nnz < 0) {
if (MyPID == 0) {
cout << "Error reading '" << filename << "'" << endl
<< "End Result: TEST FAILED" << endl;
}
return -1;
}
// create map
RCP<const Map<int> > map = rcp (new Map<int> (dim, 0, comm));
RCP<CrsMatrix<ST,int> > K = rcp(new CrsMatrix<ST,int>(map,rnnzmax));
if (MyPID == 0) {
// Convert interleaved doubles to complex values
// HB format is compressed column. CrsMatrix is compressed row.
const double *dptr = dvals;
const int *rptr = rowind;
for (int c=0; c<dim; ++c) {
for (int colnnz=0; colnnz < colptr[c+1]-colptr[c]; ++colnnz) {
K->insertGlobalValues(*rptr++ - 1,tuple(c),tuple(ST(dptr[0],dptr[1])));
dptr += 2;
}
}
}
if (MyPID == 0) {
// Clean up.
free( dvals );
free( colptr );
free( rowind );
}
K->fillComplete();
// Create initial vectors
RCP<MV> ivec = rcp( new MV(map,blockSize) );
ivec->randomize ();
// Create eigenproblem
RCP<Anasazi::BasicEigenproblem<ST,MV,OP> > problem =
rcp( new Anasazi::BasicEigenproblem<ST,MV,OP>(K,ivec) );
//
// Inform the eigenproblem that the operator K is symmetric
problem->setHermitian(true);
//
// Set the number of eigenvalues requested
problem->setNEV( nev );
//
// Inform the eigenproblem that you are done passing it information
bool boolret = problem->setProblem();
if (boolret != true) {
if (MyPID == 0) {
cout << "Anasazi::BasicEigenproblem::SetProblem() returned with error." << endl
<< "End Result: TEST FAILED" << endl;
}
return -1;
}
// Set verbosity level
int verbosity = Anasazi::Errors + Anasazi::Warnings + Anasazi::FinalSummary + Anasazi::TimingDetails;
if (verbose) {
verbosity += Anasazi::IterationDetails;
}
if (debug) {
verbosity += Anasazi::Debug;
}
// Eigensolver parameters
int maxIters = 450;
//
// Create parameter list to pass into the solver manager
ParameterList MyPL;
MyPL.set( "Skinny Solver", skinny);
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Which", which );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Maximum Iterations", maxIters );
MyPL.set( "Convergence Tolerance", tol );
//
// Create the solver manager
Anasazi::RTRSolMgr<ST,MV,OP> MySolverMgr(problem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMgr.solve();
testFailed = false;
if (returnCode != Anasazi::Converged) {
testFailed = true;
}
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<ST,MV> sol = problem->getSolution();
RCP<MV> evecs = sol.Evecs;
int numev = sol.numVecs;
if (numev > 0) {
std::ostringstream os;
os.setf(std::ios::scientific, std::ios::floatfield);
os.precision(6);
// Compute the direct residual
std::vector<MT> normV( numev );
SerialDenseMatrix<int,ST> T(numev,numev);
for (int i=0; i<numev; i++) {
T(i,i) = sol.Evals[i].realpart;
}
RCP<MV> Kvecs = MVT::Clone( *evecs, numev );
OPT::Apply( *K, *evecs, *Kvecs );
MVT::MvTimesMatAddMv( -ONE, *evecs, T, ONE, *Kvecs );
MVT::MvNorm( *Kvecs, normV );
os << "Direct residual norms computed in Tpetra_IRTR_complex_test.exe" << endl
<< std::setw(20) << "Eigenvalue" << std::setw(20) << "Residual " << endl
<< "----------------------------------------" << endl;
for (int i=0; i<numev; i++) {
if ( SCT::magnitude(sol.Evals[i].realpart) != SCT::zero() ) {
normV[i] = SCT::magnitude(normV[i]/sol.Evals[i].realpart);
}
os << std::setw(20) << sol.Evals[i].realpart << std::setw(20) << normV[i] << endl;
if ( normV[i] > tol ) {
testFailed = true;
}
}
if (MyPID==0) {
cout << endl << os.str() << endl;
}
}
if (testFailed) {
if (MyPID==0) {
cout << "End Result: TEST FAILED" << endl;
}
return -1;
}
//
// Default return value
//
if (MyPID==0) {
cout << "End Result: TEST PASSED" << endl;
}
return 0;
}
size_t computeLocalEdgeList(
const RCP<const Environment> &env, const RCP<const Comm<int> > &comm,
size_t numLocalEdges, // local edges
size_t numLocalGraphEdges, // edges in "local" graph
RCP<const IdentifierMap<User> > &idMap,
ArrayRCP<const typename InputTraits<User>::zgid_t> &allEdgeIds, // in
ArrayRCP<const typename InputTraits<User>::gno_t> &allEdgeGnos, // in
ArrayRCP<int> &allProcs, // in
ArrayRCP<const typename InputTraits<User>::lno_t> &allOffs, // in
ArrayRCP<StridedData<typename InputTraits<User>::lno_t,
typename InputTraits<User>::scalar_t> > &allWeights,// in
ArrayRCP<const typename InputTraits<User>::lno_t> &edgeLocalIds, //
ArrayRCP<const typename InputTraits<User>::lno_t> &offsets, // out
ArrayRCP<StridedData<typename InputTraits<User>::lno_t,
typename InputTraits<User>::scalar_t> > &eWeights) // out
{
typedef typename InputTraits<User>::zgid_t zgid_t;
typedef typename InputTraits<User>::gno_t gno_t;
typedef typename InputTraits<User>::scalar_t scalar_t;
typedef typename InputTraits<User>::lno_t lno_t;
typedef StridedData<lno_t, scalar_t> input_t;
int rank = comm->getRank();
bool gnosAreGids = idMap->gnosAreGids();
edgeLocalIds = ArrayRCP<const lno_t>(Teuchos::null);
eWeights = ArrayRCP<input_t>(Teuchos::null);
offsets = ArrayRCP<const lno_t>(Teuchos::null);
if (numLocalGraphEdges == 0) {
// Set the offsets array and return
size_t allOffsSize = allOffs.size();
lno_t *offs = new lno_t [allOffsSize];
env->localMemoryAssertion(__FILE__, __LINE__, allOffsSize, offs);
for (size_t i = 0; i < allOffsSize; i++) offs[i] = 0;
offsets = arcp(offs, 0, allOffsSize, true);
return 0;
}
if (numLocalGraphEdges == numLocalEdges){
// Entire graph is local.
lno_t *lnos = new lno_t [numLocalGraphEdges];
env->localMemoryAssertion(__FILE__, __LINE__, numLocalGraphEdges, lnos);
if (comm->getSize() == 1) {
// With one rank, Can use gnos as local index.
if (gnosAreGids)
for (size_t i=0; i < numLocalEdges; i++) lnos[i] = allEdgeIds[i];
else
for (size_t i=0; i < numLocalEdges; i++) lnos[i] = allEdgeGnos[i];
}
else {
ArrayRCP<gno_t> gnoArray;
if (gnosAreGids){
ArrayRCP<const gno_t> gnosConst =
arcp_reinterpret_cast<const gno_t>(allEdgeIds);
gnoArray = arcp_const_cast<gno_t>(gnosConst);
}
else {
gnoArray = arcp_const_cast<gno_t>(allEdgeGnos);
}
// Need to translate to gnos to local indexing
ArrayView<lno_t> lnoView(lnos, numLocalGraphEdges);
try {
idMap->lnoTranslate(lnoView,
gnoArray.view(0,numLocalGraphEdges),
TRANSLATE_LIB_TO_APP);
}
Z2_FORWARD_EXCEPTIONS;
}
edgeLocalIds = arcp(lnos, 0, numLocalGraphEdges, true);
offsets = allOffs;
eWeights = allWeights;
}
static Teuchos::RCP<MultiVector>
CreateCartesianCoordinates(std::string const &coordType, RCP<const Map> const & map, Teuchos::ParameterList& list) {
using Galeri::Xpetra::VectorTraits;
Teuchos::RCP<MultiVector> coordinates;
GlobalOrdinal ix, iy, iz;
Scalar delta_x, delta_y, delta_z;
Scalar lx = Teuchos::as<Scalar>(list.get<double>("lx", 1) * list.get<double>("stretchx", 1));
Scalar ly = Teuchos::as<Scalar>(list.get<double>("ly", 1) * list.get<double>("stretchy", 1));
Scalar lz = Teuchos::as<Scalar>(list.get<double>("lz", 1) * list.get<double>("stretchz", 1));
GlobalOrdinal nx = list.get<GlobalOrdinal>("nx", -1);
GlobalOrdinal ny = list.get<GlobalOrdinal>("ny", -1);
GlobalOrdinal nz = list.get<GlobalOrdinal>("nz", -1);
size_t NumMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> MyGlobalElements = map->getNodeElementList();
if (coordType == "1D") {
coordinates = VectorTraits<Map,MultiVector>::Build(map, 1, false);
Teuchos::ArrayRCP<Teuchos::ArrayRCP<Scalar> > Coord(1);
Coord[0] = coordinates->getDataNonConst(0);
delta_x = lx / Teuchos::as<Scalar>(nx - 1);
for (size_t i = 0; i < NumMyElements; ++i) {
ix = MyGlobalElements[i];
Coord[0][i] = delta_x * Teuchos::as<Scalar>(ix);
}
} else if (coordType == "2D") {
coordinates = VectorTraits<Map,MultiVector>::Build(map, 2, false);
Teuchos::ArrayRCP<Teuchos::ArrayRCP<Scalar> > Coord(2);
Coord[0] = coordinates->getDataNonConst(0);
Coord[1] = coordinates->getDataNonConst(1);
delta_x = lx / Teuchos::as<Scalar>(nx - 1);
delta_y = ly / Teuchos::as<Scalar>(ny - 1);
for (size_t i = 0; i < NumMyElements; ++i) {
ix = MyGlobalElements[i] % nx;
iy = (MyGlobalElements[i] - ix) / nx;
Coord[0][i] = delta_x * Teuchos::as<Scalar>(ix);
Coord[1][i] = delta_y * Teuchos::as<Scalar>(iy);
}
} else if (coordType == "3D") {
coordinates = VectorTraits<Map,MultiVector>::Build(map, 3, false);
Teuchos::ArrayRCP<Teuchos::ArrayRCP<Scalar> > Coord(3);
Coord[0] = coordinates->getDataNonConst(0);
Coord[1] = coordinates->getDataNonConst(1);
Coord[2] = coordinates->getDataNonConst(2);
delta_x = lx / Teuchos::as<Scalar>(nx - 1);
delta_y = ly / Teuchos::as<Scalar>(ny - 1);
delta_z = lz / Teuchos::as<Scalar>(nz - 1);
for (size_t i = 0; i < NumMyElements; i++) {
GlobalOrdinal ixy = MyGlobalElements[i] % (nx * ny);
iz = (MyGlobalElements[i] - ixy) / (nx * ny);
ix = ixy % nx;
iy = (ixy - ix) / nx;
Coord[0][i] = delta_x * Teuchos::as<Scalar>(ix);
Coord[1][i] = delta_y * Teuchos::as<Scalar>(iy);
Coord[2][i] = delta_z * Teuchos::as<Scalar>(iz);
}
} else {
throw(std::runtime_error("in Galeri::Xpetra::Utils : `coordType' has incorrect value (" + coordType + ")"));
} //if (coordType == ...
return coordinates;
} // CreateCartesianCoordinates()
int main(int argc, char *argv[]) {
int status=0; // 0 = pass, failures are incremented
bool success = true;
#ifdef ALBANY_DEBUG
Teuchos::GlobalMPISession mpiSession(&argc, &argv);
#else // bypass printing process startup info
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
#endif
Kokkos::initialize();
#ifdef ALBANY_FLUSH_DENORMALS
_MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON);
_MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
#endif
#ifdef ALBANY_CHECK_FPE
// Catch FPEs. Follow Main_SolveT.cpp's approach to checking for floating
// point exceptions.
//_mm_setcsr(_MM_MASK_MASK &~ (_MM_MASK_OVERFLOW | _MM_MASK_INVALID | _MM_MASK_DIV_ZERO) );
_MM_SET_EXCEPTION_MASK(_MM_GET_EXCEPTION_MASK() & ~_MM_MASK_INVALID);
#endif
using Teuchos::RCP;
using Teuchos::rcp;
RCP<Teuchos::FancyOStream> out(Teuchos::VerboseObjectBase::getDefaultOStream());
// Command-line argument for input file
Albany::CmdLineArgs cmd;
cmd.parse_cmdline(argc, argv, *out);
try {
RCP<Teuchos::Time> totalTime =
Teuchos::TimeMonitor::getNewTimer("Albany: ***Total Time***");
RCP<Teuchos::Time> setupTime =
Teuchos::TimeMonitor::getNewTimer("Albany: Setup Time");
Teuchos::TimeMonitor totalTimer(*totalTime); //start timer
Teuchos::TimeMonitor setupTimer(*setupTime); //start timer
RCP<const Teuchos_Comm> comm =
Tpetra::DefaultPlatform::getDefaultPlatform().getComm();
// Connect vtune for performance profiling
if (cmd.vtune) {
Albany::connect_vtune(comm->getRank());
}
Albany::SolverFactory slvrfctry(cmd.xml_filename, comm);
RCP<Epetra_Comm> appComm = Albany::createEpetraCommFromTeuchosComm(comm);
RCP<Albany::Application> app;
const RCP<Thyra::ModelEvaluator<double> > solver =
slvrfctry.createThyraSolverAndGetAlbanyApp(app, appComm, appComm);
setupTimer.~TimeMonitor();
// PHX::InitializeKokkosDevice();
Teuchos::ParameterList &solveParams =
slvrfctry.getAnalysisParameters().sublist("Solve", /*mustAlreadyExist =*/ false);
// By default, request the sensitivities if not explicitly disabled
solveParams.get("Compute Sensitivities", true);
Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<double> > > thyraResponses;
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<double> > > > thyraSensitivities;
// The PoissonSchrodinger_SchroPo and PoissonSchroMosCap1D tests seg fault as albanyApp is null -
// For now, do not resize the response vectors. FIXME sort out this issue.
if(Teuchos::nonnull(app))
Piro::PerformSolveBase(*solver, solveParams, thyraResponses, thyraSensitivities, app->getAdaptSolMgr()->getSolObserver());
else
Piro::PerformSolveBase(*solver, solveParams, thyraResponses, thyraSensitivities);
Teuchos::Array<Teuchos::RCP<const Epetra_Vector> > responses;
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Epetra_MultiVector> > > sensitivities;
epetraFromThyra(appComm, thyraResponses, thyraSensitivities, responses, sensitivities);
const int num_p = solver->Np(); // Number of *vectors* of parameters
const int num_g = solver->Ng(); // Number of *vectors* of responses
*out << "Finished eval of first model: Params, Responses "
<< std::setprecision(12) << std::endl;
Teuchos::ParameterList& parameterParams = slvrfctry.getParameters().sublist("Problem").sublist("Parameters");
int num_param_vecs = (parameterParams.isType<int>("Number")) ?
int(parameterParams.get("Number", 0) > 0) :
parameterParams.get("Number of Parameter Vectors", 0);
const Thyra::ModelEvaluatorBase::InArgs<double> nominal = solver->getNominalValues();
double norm2;
for (int i=0; i<num_p; i++) {
const Teuchos::RCP<const Epetra_Vector> p_init = epetraVectorFromThyra(appComm, nominal.get_p(i));
if(i < num_param_vecs)
p_init->Print(*out << "\nParameter vector " << i << ":\n");
else { //distributed parameters, we print only 2-norm
p_init->Norm2(&norm2);
*out << "\nDistributed Parameter " << i << ": " << norm2 << " (two-norm)\n" << std::endl;
}
}
for (int i=0; i<num_g-1; i++) {
const RCP<const Epetra_Vector> g = responses[i];
bool is_scalar = true;
if (app != Teuchos::null)
is_scalar = app->getResponse(i)->isScalarResponse();
if (is_scalar) {
g->Print(*out << "\nResponse vector " << i << ":\n");
if (num_p == 0) {
// Just calculate regression data
status += slvrfctry.checkSolveTestResults(i, 0, g.get(), NULL);
} else {
for (int j=0; j<num_p; j++) {
const RCP<const Epetra_MultiVector> dgdp = sensitivities[i][j];
if (Teuchos::nonnull(dgdp)) {
if(j < num_param_vecs) {
dgdp->Print(*out << "\nSensitivities (" << i << "," << j << "): \n");
status += slvrfctry.checkSolveTestResults(i, j, g.get(), dgdp.get());
}
else {
const Epetra_Map serial_map(-1, 1, 0, dgdp.get()->Comm());
Epetra_MultiVector norms(serial_map,dgdp->NumVectors());
// RCP<Albany::ScalarResponseFunction> response = rcp_dynamic_cast<Albany::ScalarResponseFunction>(app->getResponse(i));
// int numResponses = response->numResponses();
*out << "\nSensitivities (" << i << "," << j << ") for Distributed Parameters: (two-norm)\n";
*out << " ";
for(int ir=0; ir<dgdp->NumVectors(); ++ir) {
(*dgdp)(ir)->Norm2(&norm2);
(*norms(ir))[0] = norm2;
*out << " " << norm2;
}
*out << "\n" << std::endl;
status += slvrfctry.checkSolveTestResults(i, j, g.get(), &norms);
}
}
}
}
}
}
// Create debug output object
Teuchos::ParameterList &debugParams =
slvrfctry.getParameters().sublist("Debug Output", true);
bool writeToMatrixMarketSoln = debugParams.get("Write Solution to MatrixMarket", false);
bool writeToMatrixMarketDistrSolnMap = debugParams.get("Write Distributed Solution and Map to MatrixMarket", false);
bool writeToCoutSoln = debugParams.get("Write Solution to Standard Output", false);
const RCP<const Epetra_Vector> xfinal = responses.back();
double mnv; xfinal->MeanValue(&mnv);
*out << "Main_Solve: MeanValue of final solution " << mnv << std::endl;
*out << "\nNumber of Failed Comparisons: " << status << std::endl;
if (writeToCoutSoln == true)
std::cout << "xfinal: " << *xfinal << std::endl;
#ifdef ALBANY_PERIDIGM
#if defined(ALBANY_EPETRA)
if (Teuchos::nonnull(LCM::PeridigmManager::self())) {
*out << setprecision(12) << "\nPERIDIGM-ALBANY OPTIMIZATION-BASED COUPLING FINAL FUNCTIONAL VALUE (MAIN) = "
<< LCM::PeridigmManager::self()->obcEvaluateFunctional() << "\n" << std::endl;
}
#endif
#endif
if (debugParams.get<bool>("Analyze Memory", false))
Albany::printMemoryAnalysis(std::cout, comm);
if (writeToMatrixMarketSoln == true) {
//create serial map that puts the whole solution on processor 0
int numMyElements = (xfinal->Comm().MyPID() == 0) ? app->getDiscretization()->getMap()->NumGlobalElements() : 0;
const Epetra_Map serial_map(-1, numMyElements, 0, xfinal->Comm());
//create importer from parallel map to serial map and populate serial solution xfinal_serial
Epetra_Import importOperator(serial_map, *app->getDiscretization()->getMap());
Epetra_Vector xfinal_serial(serial_map);
xfinal_serial.Import(*app->getDiscretization()->getSolutionField(), importOperator, Insert);
//writing to MatrixMarket file
EpetraExt::MultiVectorToMatrixMarketFile("xfinal.mm", xfinal_serial);
}
if (writeToMatrixMarketDistrSolnMap == true) {
//writing to MatrixMarket file
EpetraExt::MultiVectorToMatrixMarketFile("xfinal_distributed.mm", *xfinal);
EpetraExt::BlockMapToMatrixMarketFile("xfinal_distributed_map.mm", *app->getDiscretization()->getMap());
}
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (!success) status+=10000;
Teuchos::TimeMonitor::summarize(*out,false,true,false/*zero timers*/);
Kokkos::finalize_all();
return status;
}
int main(int argc, char *argv[])
{
Teuchos::GlobalMPISession session(&argc, &argv);
RCP<const Comm<int> > comm = DefaultComm<int>::getComm();
int rank = comm->getRank();
int fail = 0, gfail=0;
// Create an object that can give us test Tpetra, Xpetra
// and Epetra graphs for testing.
RCP<UserInputForTests> uinput;
try{
uinput =
rcp(new UserInputForTests(testDataFilePath,std::string("simple"), comm, true));
}
catch(std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0, string("input ")+e.what(), 1);
}
RCP<tgraph_t> tG; // original graph (for checking)
RCP<tgraph_t> newG; // migrated graph
tG = uinput->getUITpetraCrsGraph();
size_t nvtx = tG->getNodeNumRows();
// To test migration in the input adapter we need a Solution
// object. The Solution needs an IdentifierMap.
// Our solution just assigns all objects to part zero.
typedef Zoltan2::IdentifierMap<tgraph_t> idmap_t;
RCP<const Zoltan2::Environment> env = rcp(new Zoltan2::Environment);
int nWeights = 1;
typedef Zoltan2::XpetraCrsGraphAdapter<tgraph_t> adapter_t;
typedef Zoltan2::PartitioningSolution<adapter_t> soln_t;
typedef adapter_t::part_t part_t;
part_t *p = new part_t [nvtx];
memset(p, 0, sizeof(part_t) * nvtx);
ArrayRCP<part_t> solnParts(p, 0, nvtx, true);
soln_t solution(env, comm, nWeights);
solution.setParts(solnParts);
/////////////////////////////////////////////////////////////
// User object is Tpetra::CrsGraph
if (!gfail){
RCP<const tgraph_t> ctG = rcp_const_cast<const tgraph_t>(tG);
RCP<Zoltan2::XpetraCrsGraphAdapter<tgraph_t> > tGInput;
try {
tGInput =
rcp(new Zoltan2::XpetraCrsGraphAdapter<tgraph_t>(ctG));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraCrsGraphAdapter ")+e.what(), 1);
}
if (rank==0)
std::cout << "Input adapter for Tpetra::CrsGraph" << std::endl;
fail = verifyInputAdapter<tgraph_t>(*tGInput, *tG);
gfail = globalFail(comm, fail);
if (!gfail){
tgraph_t *mMigrate = NULL;
try{
tGInput->applyPartitioningSolution( *tG, mMigrate, solution);
newG = rcp(mMigrate);
}
catch (std::exception &e){
fail = 11;
}
gfail = globalFail(comm, fail);
if (!gfail){
RCP<const tgraph_t> cnewG = rcp_const_cast<const tgraph_t>(newG);
RCP<Zoltan2::XpetraCrsGraphAdapter<tgraph_t> > newInput;
try{
newInput = rcp(new Zoltan2::XpetraCrsGraphAdapter<tgraph_t>(cnewG));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraCrsGraphAdapter 2 ")+e.what(), 1);
}
if (rank==0){
std::cout <<
"Input adapter for Tpetra::CrsGraph migrated to proc 0" <<
std::endl;
}
fail = verifyInputAdapter<tgraph_t>(*newInput, *newG);
if (fail) fail += 100;
gfail = globalFail(comm, fail);
}
}
if (gfail){
printFailureCode(comm, fail);
}
}
/////////////////////////////////////////////////////////////
// User object is Xpetra::CrsGraph
if (!gfail){
RCP<xgraph_t> xG = uinput->getUIXpetraCrsGraph();
RCP<const xgraph_t> cxG = rcp_const_cast<const xgraph_t>(xG);
RCP<Zoltan2::XpetraCrsGraphAdapter<xgraph_t> > xGInput;
try {
xGInput =
rcp(new Zoltan2::XpetraCrsGraphAdapter<xgraph_t>(cxG));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraCrsGraphAdapter 3 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Input adapter for Xpetra::CrsGraph" << std::endl;
}
fail = verifyInputAdapter<xgraph_t>(*xGInput, *tG);
gfail = globalFail(comm, fail);
if (!gfail){
xgraph_t *mMigrate =NULL;
try{
xGInput->applyPartitioningSolution( *xG, mMigrate, solution);
}
catch (std::exception &e){
fail = 11;
}
gfail = globalFail(comm, fail);
if (!gfail){
RCP<const xgraph_t> cnewG(mMigrate);
RCP<Zoltan2::XpetraCrsGraphAdapter<xgraph_t> > newInput;
try{
newInput =
rcp(new Zoltan2::XpetraCrsGraphAdapter<xgraph_t>(cnewG));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraCrsGraphAdapter 4 ")+e.what(), 1);
}
if (rank==0){
std::cout <<
"Input adapter for Xpetra::CrsGraph migrated to proc 0" <<
std::endl;
}
fail = verifyInputAdapter<xgraph_t>(*newInput, *newG);
if (fail) fail += 100;
gfail = globalFail(comm, fail);
}
}
if (gfail){
printFailureCode(comm, fail);
}
}
#ifdef HAVE_EPETRA_DATA_TYPES
/////////////////////////////////////////////////////////////
// User object is Epetra_CrsGraph
if (!gfail){
RCP<egraph_t> eG = uinput->getUIEpetraCrsGraph();
RCP<const egraph_t> ceG = rcp_const_cast<const egraph_t>(eG);
RCP<Zoltan2::XpetraCrsGraphAdapter<egraph_t> > eGInput;
try {
eGInput =
rcp(new Zoltan2::XpetraCrsGraphAdapter<egraph_t>(ceG));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraCrsGraphAdapter 5 ")+e.what(), 1);
}
if (rank==0){
std::cout << "Input adapter for Epetra_CrsGraph" << std::endl;
}
fail = verifyInputAdapter<egraph_t>(*eGInput, *tG);
gfail = globalFail(comm, fail);
if (!gfail){
egraph_t *mMigrate =NULL;
try{
eGInput->applyPartitioningSolution( *eG, mMigrate, solution);
}
catch (std::exception &e){
fail = 11;
}
gfail = globalFail(comm, fail);
if (!gfail){
RCP<const egraph_t> cnewG(mMigrate, true);
RCP<Zoltan2::XpetraCrsGraphAdapter<egraph_t> > newInput;
try{
newInput =
rcp(new Zoltan2::XpetraCrsGraphAdapter<egraph_t>(cnewG));
}
catch (std::exception &e){
TEST_FAIL_AND_EXIT(*comm, 0,
string("XpetraCrsGraphAdapter 6 ")+e.what(), 1);
}
if (rank==0){
std::cout <<
"Input adapter for Epetra_CrsGraph migrated to proc 0" <<
std::endl;
}
fail = verifyInputAdapter<egraph_t>(*newInput, *newG);
if (fail) fail += 100;
gfail = globalFail(comm, fail);
}
}
if (gfail){
printFailureCode(comm, fail);
}
}
#endif
/////////////////////////////////////////////////////////////
// DONE
if (rank==0)
std::cout << "PASS" << std::endl;
}
int main(int argc, char *argv[])
{
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::Comm;
using Teuchos::ParameterList;
//
// Get the communicator
//
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
auto comm = Tpetra::DefaultPlatform::getDefaultPlatform().getComm();
const int myImageID = comm->getRank();
//
// Get example parameters from command-line processor
//
bool verbose = (myImageID==0);
bool unfused = false;
std::string matfile;
std::string xmlfile;
std::string machineFile;
Teuchos::CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("matrix-file",&matfile,"Filename for matrix");
cmdp.setOption("param-file", &xmlfile,"XML file for solver parameters");
cmdp.setOption("machine-file",&machineFile,"Filename for XML machine description file.");
cmdp.setOption("unfused","no-unfused",&unfused,"Test unfused iteration.");
if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
return -1;
}
//
// read machine file and initialize platform
//
RCP<Teuchos::ParameterList> machinePL = Teuchos::parameterList();
std::string defaultMachine(
" <ParameterList> "
" <ParameterList name='%1=0'> "
" <Parameter name='NodeType' type='string' value='Kokkos::SerialNode'/> "
" </ParameterList> "
" </ParameterList> "
);
Teuchos::updateParametersFromXmlString(defaultMachine,machinePL.ptr());
if (machineFile != "") Teuchos::updateParametersFromXmlFile(machineFile,machinePL.ptr());
//
// create the platform object
//
Tpetra::HybridPlatform platform(comm,*machinePL);
//
// Define the type stack
//
TPETRAEXT_TYPESTACK2(MPStack, qd_real, dd_real )
//
// instantiate a driver on the scalar stack
//
MultiPrecDriver<MPStack> driver;
// hand output stream to driver
if (verbose) driver.out = Teuchos::getFancyOStream(Teuchos::rcp(&std::cout,false));
else driver.out = Teuchos::getFancyOStream(Teuchos::rcp(new Teuchos::oblackholestream()));
// hand matrix file to driver
driver.matrixFile = matfile;
// other params
driver.unfusedTest = unfused;
//
// get the solver parameters
//
RCP<Teuchos::ParameterList> params = Teuchos::parameterList();
// default solver stack parameters
std::string xmlString(
" <ParameterList> \n"
" <Parameter name='tolerance' value='1e-60' type='double'/> \n"
" <Parameter name='verbose' value='2' type='int'/> \n"
" <ParameterList name='child'> \n"
" <Parameter name='tolerance' value='1e-28' type='double'/> \n"
" <Parameter name='verbose' value='2' type='int'/> \n"
" <Parameter name='Extract Diagonal' value='true' type='bool'/> \n"
" </ParameterList> \n"
" </ParameterList> \n"
);
Teuchos::updateParametersFromXmlString(xmlString,params.ptr());
if (xmlfile != "") Teuchos::updateParametersFromXmlFile(xmlfile,params.ptr());
// hand solver parameters to driver
driver.params = params;
//
// run the driver
//
platform.runUserCode(driver);
//
// Print result
if (driver.testPassed) {
*driver.out << "End Result: TEST PASSED" << std::endl;
}
return 0;
}
TEUCHOS_UNIT_TEST(assembly_engine, basic_tpetra)
{
using Teuchos::RCP;
// build global communicator
Teuchos::RCP<Teuchos::Comm<int> > comm = Teuchos::rcp(new Teuchos::MpiComm<int>(Teuchos::opaqueWrapper(MPI_COMM_WORLD)));
RCP<Teuchos::ParameterList> pl = rcp(new Teuchos::ParameterList);
pl->set("X Blocks",2);
pl->set("Y Blocks",1);
pl->set("X Elements",6);
pl->set("Y Elements",4);
panzer_stk_classic::SquareQuadMeshFactory factory;
factory.setParameterList(pl);
RCP<panzer_stk_classic::STK_Interface> mesh = factory.buildMesh(MPI_COMM_WORLD);
Teuchos::RCP<Teuchos::ParameterList> ipb = Teuchos::parameterList("Physics Blocks");
std::vector<panzer::BC> bcs;
testInitialzation(ipb, bcs);
Teuchos::RCP<panzer::FieldManagerBuilder> fmb =
Teuchos::rcp(new panzer::FieldManagerBuilder);
// build physics blocks
//////////////////////////////////////////////////////////////
const std::size_t workset_size = 20;
Teuchos::RCP<user_app::MyFactory> eqset_factory = Teuchos::rcp(new user_app::MyFactory);
user_app::BCFactory bc_factory;
std::vector<Teuchos::RCP<panzer::PhysicsBlock> > physicsBlocks;
{
const int default_integration_order = 1;
std::map<std::string,std::string> block_ids_to_physics_ids;
block_ids_to_physics_ids["eblock-0_0"] = "test physics";
block_ids_to_physics_ids["eblock-1_0"] = "test physics";
std::map<std::string,Teuchos::RCP<const shards::CellTopology> > block_ids_to_cell_topo;
block_ids_to_cell_topo["eblock-0_0"] = mesh->getCellTopology("eblock-0_0");
block_ids_to_cell_topo["eblock-1_0"] = mesh->getCellTopology("eblock-1_0");
Teuchos::RCP<panzer::GlobalData> gd = panzer::createGlobalData();
panzer::buildPhysicsBlocks(block_ids_to_physics_ids,
block_ids_to_cell_topo,
ipb,
default_integration_order,
workset_size,
eqset_factory,
gd,
false,
physicsBlocks);
}
// build worksets
//////////////////////////////////////////////////////////////
Teuchos::RCP<panzer_stk_classic::WorksetFactory> wkstFactory
= Teuchos::rcp(new panzer_stk_classic::WorksetFactory(mesh)); // build STK workset factory
Teuchos::RCP<panzer::WorksetContainer> wkstContainer // attach it to a workset container (uses lazy evaluation)
= Teuchos::rcp(new panzer::WorksetContainer(wkstFactory,physicsBlocks,workset_size));
// build DOF Manager
/////////////////////////////////////////////////////////////
// build the connection manager
const Teuchos::RCP<panzer::ConnManager<int,panzer::Ordinal64> >
conn_manager = Teuchos::rcp(new panzer_stk_classic::STKConnManager<panzer::Ordinal64>(mesh));
panzer::DOFManagerFactory<int,panzer::Ordinal64> globalIndexerFactory;
RCP<panzer::UniqueGlobalIndexer<int,panzer::Ordinal64> > dofManager
= globalIndexerFactory.buildUniqueGlobalIndexer(Teuchos::opaqueWrapper(MPI_COMM_WORLD),physicsBlocks,conn_manager);
Teuchos::RCP<panzer::LinearObjFactory<panzer::Traits> > linObjFactory
= Teuchos::rcp(new panzer::TpetraLinearObjFactory<panzer::Traits,double,int,panzer::Ordinal64>(comm,dofManager));
// setup field manager build
/////////////////////////////////////////////////////////////
// Add in the application specific closure model factory
panzer::ClosureModelFactory_TemplateManager<panzer::Traits> cm_factory;
user_app::MyModelFactory_TemplateBuilder cm_builder;
cm_factory.buildObjects(cm_builder);
Teuchos::ParameterList closure_models("Closure Models");
closure_models.sublist("solid").sublist("SOURCE_TEMPERATURE").set<double>("Value",1.0);
closure_models.sublist("ion solid").sublist("SOURCE_ION_TEMPERATURE").set<double>("Value",1.0);
Teuchos::ParameterList user_data("User Data");
fmb->setWorksetContainer(wkstContainer);
fmb->setupVolumeFieldManagers(physicsBlocks,cm_factory,closure_models,*linObjFactory,user_data);
fmb->setupBCFieldManagers(bcs,physicsBlocks,*eqset_factory,cm_factory,bc_factory,closure_models,*linObjFactory,user_data);
panzer::AssemblyEngine_TemplateManager<panzer::Traits> ae_tm;
panzer::AssemblyEngine_TemplateBuilder builder(fmb,linObjFactory);
ae_tm.buildObjects(builder);
RCP<panzer::LinearObjContainer> tGhosted = linObjFactory->buildGhostedLinearObjContainer();
RCP<panzer::LinearObjContainer> tGlobal = linObjFactory->buildLinearObjContainer();
linObjFactory->initializeGhostedContainer(panzer::LinearObjContainer::X |
panzer::LinearObjContainer::DxDt |
panzer::LinearObjContainer::F |
panzer::LinearObjContainer::Mat,*tGhosted);
linObjFactory->initializeContainer(panzer::EpetraLinearObjContainer::X |
panzer::EpetraLinearObjContainer::DxDt |
panzer::EpetraLinearObjContainer::F |
panzer::EpetraLinearObjContainer::Mat,*tGlobal);
// panzer::pauseToAttach();
tGhosted->initialize();
tGlobal->initialize();
panzer::AssemblyEngineInArgs input(tGhosted,tGlobal);
input.alpha = 0.0;
input.beta = 1.0;
ae_tm.getAsObject<panzer::Traits::Residual>()->evaluate(input);
ae_tm.getAsObject<panzer::Traits::Jacobian>()->evaluate(input);
RCP<panzer::TpetraLinearObjContainer<double,int,panzer::Ordinal64> > globalCont
= Teuchos::rcp_dynamic_cast<panzer::TpetraLinearObjContainer<double,int,panzer::Ordinal64> >(tGlobal);
Teuchos::RCP<const Tpetra::Operator<double,int,panzer::Ordinal64> > baseOp = globalCont->get_A();
Teuchos::RCP<const Thyra::VectorSpaceBase<double> > rangeSpace = Thyra::createVectorSpace<double>(baseOp->getRangeMap());
Teuchos::RCP<const Thyra::VectorSpaceBase<double> > domainSpace = Thyra::createVectorSpace<double>(baseOp->getDomainMap());
tLinearOp = Thyra::constTpetraLinearOp<double,int,panzer::Ordinal64>(rangeSpace, domainSpace, baseOp);
tVector = Thyra::constTpetraVector<double,int,panzer::Ordinal64>(Thyra::tpetraVectorSpace<double,int,panzer::Ordinal64>(baseOp->getRangeMap()).getConst(),
globalCont->get_f().getConst());
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::Time;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false);
GO nx, ny, nz;
nx=50;
ny=50;
nz=50;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
int optNraps = 5; clp.setOption("nraps", &optNraps, "number of RAPS to perform");
bool optTimings = true; clp.setOption("timings", "notimings", &optTimings, "print timings to screen");
bool optImplicitTranspose = true; clp.setOption("implicit", "explicit", &optImplicitTranspose, "whether to form R implicitly");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
//
// Construct the problem
//
{
TimeMonitor globalTimeMonitor(*TimeMonitor::getNewTimer("RAPScalingTest: S - Global Time"));
RCP<Matrix> A;
RCP<MultiVector> coordinates;
{
TimeMonitor tm(*TimeMonitor::getNewTimer("RAPScalingTest: 1 - Matrix creation"));
RCP<const Map> map;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx", nx));
galeriList.set("ny", pl.get("ny", ny));
galeriList.set("nz", pl.get("nz", nz));
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
//FIXME
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
A = Pr->BuildMatrix();
}
Level fineLevel, coarseLevel;
RAPFactory AcFact;
AcFact.DisableMultipleCallCheck();
RCP<SaPFactory> PFact;
RCP<TransPFactory> RFact;
{
TimeMonitor tm(*TimeMonitor::getNewTimer("RAPScalingTest: 2 - Setup"));
MueLuTests::TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createTwoLevelHierarchy(fineLevel, coarseLevel); // set a default FactoryManager
fineLevel.Set("A", A);
PFact = rcp(new SaPFactory());
coarseLevel.Request("P", PFact.get());
PFact->Build(fineLevel, coarseLevel);
RFact = rcp(new TransPFactory());
AcFact.SetFactory("P", PFact);
if (optImplicitTranspose==false)
AcFact.SetFactory("R", RFact);
AcFact.SetImplicitTranspose(optImplicitTranspose);
}
RCP<Time> RAPKernelTimer = TimeMonitor::getNewTimer("RAPScalingTest: 3 - RAP kernel"); // re-use the same timer in the loop
for (int i=0; i<optNraps; ++i) {
coarseLevel.Request("A", &AcFact);
{
TimeMonitor tm(*RAPKernelTimer);
if (optImplicitTranspose==false) {
RFact->SetFactory("P", PFact);
coarseLevel.Request("R", RFact.get());
RFact->Build(fineLevel, coarseLevel);
}
RCP<Matrix> Ac = coarseLevel.Get< RCP<Matrix> >("A", &AcFact);
}
coarseLevel.Release("A", &AcFact);
if (optImplicitTranspose==false)
coarseLevel.Release("R", RFact.get());
}
} // end of globalTimeMonitor
if (optTimings) {
Teuchos::TableFormat &format = TimeMonitor::format();
format.setPrecision(25);
TimeMonitor::summarize();
}
} //main
void Piro::RythmosSolver<Scalar>::initialize(
const Teuchos::RCP<Teuchos::ParameterList> &appParams,
const Teuchos::RCP< Thyra::ModelEvaluator<Scalar> > &in_model,
const Teuchos::RCP<Rythmos::IntegrationObserverBase<Scalar> > &observer)
{
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
// set some internals
model = in_model;
num_p = in_model->Np();
num_g = in_model->Ng();
//
*out << "\nA) Get the base parameter list ...\n";
//
RCP<Teuchos::ParameterList> rythmosPL = sublist(appParams, "Rythmos", true);
rythmosPL->validateParameters(*getValidRythmosParameters(),0);
{
const std::string verbosity = rythmosPL->get("Verbosity Level", "VERB_DEFAULT");
if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE;
else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT;
else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW;
else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM;
else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH;
else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME;
else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Unknown verbosity option specified in Piro_RythmosSolver.");
}
t_initial = rythmosPL->get("Initial Time", 0.0);
t_final = rythmosPL->get("Final Time", 0.1);
const std::string stepperType = rythmosPL->get("Stepper Type", "Backward Euler");
//
*out << "\nC) Create and initalize the forward model ...\n";
//
*out << "\nD) Create the stepper and integrator for the forward problem ...\n";
//
if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "Rythmos") {
Teuchos::RCP<Rythmos::TimeStepNonlinearSolver<Scalar> > rythmosTimeStepSolver =
Rythmos::timeStepNonlinearSolver<Scalar>();
if (rythmosPL->getEntryPtr("NonLinear Solver")) {
RCP<Teuchos::ParameterList> nonlinePL =
sublist(rythmosPL, "NonLinear Solver", true);
rythmosTimeStepSolver->setParameterList(nonlinePL);
}
fwdTimeStepSolver = rythmosTimeStepSolver;
}
else if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "NOX") {
#ifdef Piro_ENABLE_NOX
Teuchos::RCP<Thyra::NOXNonlinearSolver> nox_solver = Teuchos::rcp(new Thyra::NOXNonlinearSolver);
Teuchos::RCP<Teuchos::ParameterList> nox_params = Teuchos::rcp(new Teuchos::ParameterList);
*nox_params = appParams->sublist("NOX");
nox_solver->setParameterList(nox_params);
fwdTimeStepSolver = nox_solver;
#else
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Requested NOX solver for a Rythmos Transient solve, Trilinos was not built with NOX enabled. Please rebuild Trilinos or use the native Rythmos nonlinear solver.");
#endif
}
if (stepperType == "Backward Euler") {
fwdStateStepper = Rythmos::backwardEulerStepper<Scalar> (model, fwdTimeStepSolver);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "Forward Euler") {
fwdStateStepper = Rythmos::forwardEulerStepper<Scalar> (model);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "Explicit RK") {
fwdStateStepper = Rythmos::explicitRKStepper<Scalar>(model);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "BDF") {
Teuchos::RCP<Teuchos::ParameterList> BDFparams =
Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
Teuchos::RCP<Teuchos::ParameterList> BDFStepControlPL =
Teuchos::sublist(BDFparams,"Step Control Settings");
fwdStateStepper = Teuchos::rcp( new Rythmos::ImplicitBDFStepper<Scalar>(model,fwdTimeStepSolver,BDFparams) );
fwdStateStepper->setInitialCondition(model->getNominalValues());
}
else {
// first (before failing) check to see if the user has added stepper factory
typename std::map<std::string,Teuchos::RCP<RythmosStepperFactory<Scalar> > >::const_iterator
stepFactItr = stepperFactories.find(stepperType);
if(stepFactItr!=stepperFactories.end()) {
// the user has added it, hot dog lets build a new stepper!
Teuchos::RCP<Teuchos::ParameterList> stepperParams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
// build the stepper using the factory
fwdStateStepper = stepFactItr->second->buildStepper(model,fwdTimeStepSolver,stepperParams);
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(
true, Teuchos::Exceptions::InvalidParameter,
std::endl << "Error! Piro::Epetra::RythmosSolver: Invalid Steper Type: "
<< stepperType << std::endl);
}
}
// Step control strategy
{
// If the stepper can accept a step control strategy, then attempt to build one.
RCP<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> > scsa_stepper =
Teuchos::rcp_dynamic_cast<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> >(fwdStateStepper);
if (Teuchos::nonnull(scsa_stepper)) {
const std::string step_control_strategy = rythmosPL->get("Step Control Strategy Type", "None");
if (step_control_strategy == "None") {
// don't do anything, stepper will build default
} else if (step_control_strategy == "ImplicitBDFRamping") {
const RCP<Rythmos::ImplicitBDFStepperRampingStepControl<Scalar> > rscs =
rcp(new Rythmos::ImplicitBDFStepperRampingStepControl<Scalar>);
const RCP<ParameterList> p = parameterList(rythmosPL->sublist("Rythmos Step Control Strategy"));
rscs->setParameterList(p);
scsa_stepper->setStepControlStrategy(rscs);
} else {
TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
"Error! Piro::Epetra::RythmosSolver: Invalid step control strategy type: "
<< step_control_strategy << std::endl);
}
}
}
{
const RCP<Teuchos::ParameterList> integrationControlPL =
Teuchos::sublist(rythmosPL, "Rythmos Integration Control", true);
RCP<Rythmos::DefaultIntegrator<Scalar> > defaultIntegrator;
if (rythmosPL->get("Rythmos Integration Control Strategy", "Simple") == "Simple") {
defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL));
}
else if(rythmosPL->get<std::string>("Rythmos Integration Control Strategy") == "Ramping") {
defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::rampingIntegrationControlStrategy<Scalar>(integrationControlPL));
}
fwdStateIntegrator = defaultIntegrator;
}
fwdStateIntegrator->setParameterList(sublist(rythmosPL, "Rythmos Integrator", true));
if (Teuchos::nonnull(observer)) {
fwdStateIntegrator->setIntegrationObserver(observer);
}
isInitialized = true;
}
bool tLSCIntegrationTest::test_nomassStable(int verbosity,std::ostream & os)
{
Teuchos::ParameterList paramList;
solveList(paramList,8);
RCP<Teko::InverseFactory> invFact = Teko::invFactoryFromParamList(paramList,"ML");
TEUCHOS_ASSERT(invFact!=Teuchos::null);
bool status = false;
bool allPassed = true;
// int vcycles = 8;
// load everything
loadStableSystem();
// if you get here you automatically pass!
if(verbosity>=10 ) {
os << std::endl << " tLSCIntegrationTest::test_nomassStable: loading system ... "
<< toString(true) << std::endl;
}
const RCP<Teko::NS::LSCStrategy> strategy = rcp(new Teko::NS::InvLSCStrategy(invFact));
const RCP<Teko::BlockPreconditionerFactory> precFact = rcp(new Teko::NS::LSCPreconditionerFactory(strategy));
const RCP<Teko::Epetra::EpetraBlockPreconditioner> prec = rcp(new Teko::Epetra::EpetraBlockPreconditioner(precFact));
prec->buildPreconditioner(sA_);
// B. Build solver and solve system
Epetra_Vector x(sA_->OperatorDomainMap());
x.Scale(0.0);
// build Epetra problem
Epetra_LinearProblem problem(&*sA_,&x,&*rhs_); // this doesn't take const arguments!
AztecOO solver(problem);
solver.SetAztecOption(AZ_solver,AZ_gmres);
solver.SetAztecOption(AZ_precond,AZ_none);
solver.SetAztecOption(AZ_kspace,50);
solver.SetAztecOption(AZ_output,AZ_none);
solver.SetPrecOperator(&*prec);
solver.Iterate(1000,1e-8);
// check iteration count
status = (solver.NumIters()<=43);
if(not status || verbosity>=10 ) {
os << std::endl << " tLSCIntegrationTest::test_nomassStable " << toString(status)
<< ": # of iterations = " << solver.NumIters() << " != " << 43 << std::endl;
}
allPassed &= status;
// check exact answer (versus IFISS solution)
x.Update(-1.0,*sExact_,1.0); // x = x - x*
double errnorm,exactnorm,relerr;
x.Norm2(&errnorm);
sExact_->Norm2(&exactnorm);
status = ((relerr = errnorm/exactnorm) <= tolerance_);
if(not status || verbosity==10 ) {
os << std::endl << " tLSCIntegrationTest::test_nomassStable " << toString(status)
<< ": error in solution = " << std::scientific << relerr << std::endl;
}
allPassed &= status;
return allPassed;
}
void TopSmootherFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level & level) const {
// TODO: get rid of these
typedef MueLu::SmootherBase<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> SmootherBase2_type;
typedef MueLu::SmootherFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> SmootherFactory_type;
typedef MueLu::SmootherPrototype<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> SmootherPrototype_type;
if (preSmootherFact_.is_null() && postSmootherFact_.is_null())
return;
// NOTE 1: We need to set at least some keep flag for the smoothers, otherwise it is going to be removed as soon as all requests are released.
// We choose to set the Final flag for the data. In addition, we allow this data to be retrieved by only using the name by the means
// of using NoFactory. However, any data set with NoFactory gets UserData flag by default. We don't really want that flag, so we remove it.
// NOTE 2: some smoother factories are tricky (see comments in MueLu::SmootherFactory
// Sometimes, we don't know whether the factory is able to generate "PreSmoother" or "PostSmoother"
// For the SmootherFactory, however, we are able to check that.
if (!preSmootherFact_.is_null()) {
// Checking for null is not sufficient, as SmootherFactory(null, something) does not generate "PreSmoother"
bool isAble = true;
RCP<const SmootherFactory_type> s = rcp_dynamic_cast<const SmootherFactory_type>(preSmootherFact_);
if (!s.is_null()) {
RCP<SmootherPrototype_type> pre, post;
s->GetSmootherPrototypes(pre, post);
if (pre.is_null())
isAble = false;
} else {
// We assume that if presmoother factory is not SmootherFactory, it *is* able to generate "PreSmoother"
}
if (isAble) {
RCP<SmootherBase2_type> Pre = level.Get<RCP<SmootherBase2_type> >("PreSmoother", preSmootherFact_.get());
level.Set ("PreSmoother", Pre, NoFactory::get());
level.AddKeepFlag ("PreSmoother", NoFactory::get(), MueLu::Final);
level.RemoveKeepFlag("PreSmoother", NoFactory::get(), MueLu::UserData);
}
}
if (!postSmootherFact_.is_null()) {
// Checking for null is not sufficient, as SmootherFactory(something, null) does not generate "PostSmoother"
bool isAble = true;
RCP<const SmootherFactory_type> s = rcp_dynamic_cast<const SmootherFactory_type>(postSmootherFact_);
if (!s.is_null()) {
RCP<SmootherPrototype_type> pre, post;
s->GetSmootherPrototypes(pre, post);
if (post.is_null())
isAble = false;
} else {
// We assume that if presmoother factory is not SmootherFactory, it *is* able to generate "PreSmoother"
}
if (isAble) {
RCP<SmootherBase2_type> Post = level.Get<RCP<SmootherBase2_type> >("PostSmoother", postSmootherFact_.get());
level.Set ("PostSmoother", Post, NoFactory::get());
level.AddKeepFlag ("PostSmoother", NoFactory::get(), MueLu::Final);
level.RemoveKeepFlag("PostSmoother", NoFactory::get(), MueLu::UserData);
}
}
}
void ZoltanInterface<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level& level) const {
FactoryMonitor m(*this, "Build", level);
RCP<Matrix> A = Get< RCP<Matrix> > (level, "A");
RCP<const Map> rowMap = A->getRowMap();
RCP<MultiVector> Coords = Get< RCP<MultiVector> >(level, "Coordinates");
size_t dim = Coords->getNumVectors();
GO numParts = level.Get<GO>("number of partitions");
if (numParts == 1) {
// Running on one processor, so decomposition is the trivial one, all zeros.
RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
Set(level, "Partition", decomposition);
return;
}
float zoltanVersion_;
Zoltan_Initialize(0, NULL, &zoltanVersion_);
RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
RCP<Zoltan> zoltanObj_ = rcp(new Zoltan((*zoltanComm)())); //extract the underlying MPI_Comm handle and create a Zoltan object
if (zoltanObj_ == Teuchos::null)
throw Exceptions::RuntimeError("MueLu::Zoltan : Unable to create Zoltan data structure");
// Tell Zoltan what kind of local/global IDs we will use.
// In our case, each GID is two ints and there are no local ids.
// One can skip this step if the IDs are just single ints.
int rv;
if ((rv = zoltanObj_->Set_Param("num_gid_entries", "1")) != ZOLTAN_OK)
throw Exceptions::RuntimeError("MueLu::Zoltan::Setup : setting parameter 'num_gid_entries' returned error code " + Teuchos::toString(rv));
if ((rv = zoltanObj_->Set_Param("num_lid_entries", "0") ) != ZOLTAN_OK)
throw Exceptions::RuntimeError("MueLu::Zoltan::Setup : setting parameter 'num_lid_entries' returned error code " + Teuchos::toString(rv));
if ((rv = zoltanObj_->Set_Param("obj_weight_dim", "1") ) != ZOLTAN_OK)
throw Exceptions::RuntimeError("MueLu::Zoltan::Setup : setting parameter 'obj_weight_dim' returned error code " + Teuchos::toString(rv));
if (GetVerbLevel() & Statistics1) zoltanObj_->Set_Param("debug_level", "1");
else zoltanObj_->Set_Param("debug_level", "0");
zoltanObj_->Set_Param("num_global_partitions", toString(numParts));
zoltanObj_->Set_Num_Obj_Fn(GetLocalNumberOfRows, (void *) &*A);
zoltanObj_->Set_Obj_List_Fn(GetLocalNumberOfNonzeros, (void *) &*A);
zoltanObj_->Set_Num_Geom_Fn(GetProblemDimension, (void *) &dim);
zoltanObj_->Set_Geom_Multi_Fn(GetProblemGeometry, (void *) Coords.get());
// Data pointers that Zoltan requires.
ZOLTAN_ID_PTR import_gids = NULL; // Global nums of objs to be imported
ZOLTAN_ID_PTR import_lids = NULL; // Local indices to objs to be imported
int *import_procs = NULL; // Proc IDs of procs owning objs to be imported.
int *import_to_part = NULL; // Partition #s to which imported objs should be assigned.
ZOLTAN_ID_PTR export_gids = NULL; // Global nums of objs to be exported
ZOLTAN_ID_PTR export_lids = NULL; // local indices to objs to be exported
int *export_procs = NULL; // Proc IDs of destination procs for objs to be exported.
int *export_to_part = NULL; // Partition #s for objs to be exported.
int num_imported; // Number of objs to be imported.
int num_exported; // Number of objs to be exported.
int newDecomp; // Flag indicating whether the decomposition has changed
int num_gid_entries; // Number of array entries in a global ID.
int num_lid_entries;
{
SubFactoryMonitor m1(*this, "Zoltan RCB", level);
rv = zoltanObj_->LB_Partition(newDecomp, num_gid_entries, num_lid_entries,
num_imported, import_gids, import_lids, import_procs, import_to_part,
num_exported, export_gids, export_lids, export_procs, export_to_part);
if (rv == ZOLTAN_FATAL)
throw Exceptions::RuntimeError("Zoltan::LB_Partition() returned error code");
}
// TODO check that A's row map is 1-1. Zoltan requires this.
RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition;
if (newDecomp) {
decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, false); // Don't initialize, will be overwritten
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
int mypid = rowMap->getComm()->getRank();
for (typename ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = mypid;
LO blockSize = A->GetFixedBlockSize();
for (int i = 0; i < num_exported; ++i) {
// We have assigned Zoltan gids to first row GID in the block
// NOTE: Zoltan GIDs are different from GIDs in the Coordinates vector
LO localEl = rowMap->getLocalElement(export_gids[i]);
int partNum = export_to_part[i];
for (LO j = 0; j < blockSize; ++j)
decompEntries[localEl + j] = partNum;
}
}
Set(level, "Partition", decomposition);
zoltanObj_->LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_to_part);
zoltanObj_->LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_to_part);
} //Build()
Teuchos::RCP<const PHX::FieldTag>
ShallowWaterProblemNoAD::constructEvaluators<PHAL::AlbanyTraits::Jacobian>(
PHX::FieldManager<PHAL::AlbanyTraits>& fm0,
const Albany::MeshSpecsStruct& meshSpecs,
Albany::StateManager& stateMgr,
Albany::FieldManagerChoice fieldManagerChoice,
const Teuchos::RCP<Teuchos::ParameterList>& responseList)
{
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::ParameterList;
using PHX::DataLayout;
using PHX::MDALayout;
using std::vector;
using std::string;
using std::map;
using PHAL::AlbanyTraits;
typedef PHAL::AlbanyTraits::Jacobian EvalT;
// Teuchos::RCP<Teuchos::FancyOStream> out(Teuchos::VerboseObjectBase::getDefaultOStream());
// *out << "Aeras::ShallowWaterProblemNoAD Jacobian specialization of constructEvaluators" << std::endl;
RCP<Intrepid2::Basis<PHX::Device, RealType, RealType> >
intrepidBasis = Albany::getIntrepid2Basis(meshSpecs.ctd);
RCP<shards::CellTopology> cellType = rcp(new shards::CellTopology (&meshSpecs.ctd));
const int numNodes = intrepidBasis->getCardinality();
const int worksetSize = meshSpecs.worksetSize;
// RCP <Intrepid2::Polylib<RealType, Kokkos::DynRankView<RealType, PHX::Device> > > polylib = rcp(new Intrepid2::Polylib<RealType, Kokkos::DynRankView<RealType, PHX::Device> >(meshSpecs.cubatureDegree, meshSpecs.cubatureRule));
// std::vector< Teuchos::RCP<Intrepid2::Cubature<PHX::Device> > > cubatures(2, polylib);
// RCP <Intrepid2::Cubature<PHX::Device> > cubature = rcp( new Intrepid2::CubatureTensor<RealType,Kokkos::DynRankView<RealType, PHX::Device> >(cubatures));
// Regular Gauss Quadrature.
Intrepid2::DefaultCubatureFactory cubFactory;
RCP <Intrepid2::Cubature<PHX::Device> > cubature = cubFactory.create<PHX::Device, RealType, RealType>(*cellType, meshSpecs.cubatureDegree, meshSpecs.cubatureRule);
const int numQPts = cubature->getNumPoints();
const int numVertices = meshSpecs.ctd.node_count;
int vecDim = neq;
/*if (neq == 1 || neq == 2)
vecDim = neq;
else
vecDim = spatialDim;
*/
*out << "Field Dimensions: Workset=" << worksetSize
<< ", Vertices= " << numVertices
<< ", Nodes= " << numNodes
<< ", QuadPts= " << numQPts
<< ", Spatial Dim= " << spatialDim
<< ", Model Dim= " << modelDim
<< ", vecDim= " << vecDim << std::endl;
dl = rcp(new Aeras::Layouts(worksetSize,numVertices,numNodes,numQPts, modelDim, vecDim, 0));
Albany::EvaluatorUtils<EvalT, PHAL::AlbanyTraits> evalUtils(dl);
// Temporary variable used numerous times below
Teuchos::RCP<PHX::Evaluator<AlbanyTraits> > ev;
// Define Field Names
Teuchos::ArrayRCP<std::string> dof_names(1);
Teuchos::ArrayRCP<std::string> dof_names_dummy(1);
Teuchos::ArrayRCP<std::string> dof_names_dot(1);
Teuchos::ArrayRCP<std::string> dof_names_dotdot(1);
Teuchos::ArrayRCP<std::string> resid_names(1);
dof_names[0] = "Flow State";
dof_names_dummy[0] = "Flow State Dummy";
dof_names_dot[0] = dof_names[0]+"_dot";
dof_names_dotdot[0] = dof_names[0]+"_dotdot";
resid_names[0] = "ShallowWater Residual";
//IKT: this is the equivalent of the supportsTransient flag in LCM.
//It tells the code to build 2nd derivative terms, which we need for
//explicit integration of the system with hyperviscosity.
//TODO? set this to off when it's not needed (i.e., when no hyperviscosity
//and/or implicit time stepping).
bool explicitHV = true;
// Construct Standard FEM evaluators for Vector equation
fm0.template registerEvaluator<EvalT>
(evalUtils.constructGatherSolutionEvaluator(true, dof_names, dof_names_dot));
if (explicitHV) fm0.template registerEvaluator<EvalT>
(evalUtils.constructGatherSolutionEvaluator_withAcceleration(true, dof_names_dummy, Teuchos::null, dof_names_dotdot));
if (explicitHV) fm0.template registerEvaluator<EvalT>
(evalUtils.constructDOFVecInterpolationEvaluator(dof_names_dotdot[0]));
fm0.template registerEvaluator<EvalT>
(evalUtils.constructDOFVecInterpolationEvaluator(dof_names[0]));
fm0.template registerEvaluator<EvalT>
(evalUtils.constructDOFVecInterpolationEvaluator(dof_names_dot[0]));
fm0.template registerEvaluator<EvalT>
(evalUtils.constructDOFVecGradInterpolationEvaluator(dof_names[0]));
fm0.template registerEvaluator<EvalT>
(evalUtils.constructScatterResidualEvaluator(true, resid_names, 0, "Scatter ShallowWater"));
// Shells: 3 coords for 2D topology
if (spatialDim != modelDim) {
RCP<ParameterList> p = rcp(new ParameterList("Gather Coordinate Vector"));
// Input:
// Output:: Coordindate Vector at vertices
p->set<string>("Coordinate Vector Name", "Coord Vec");
ev = rcp(new Aeras::GatherCoordinateVector<EvalT,AlbanyTraits>(*p,dl));
fm0.template registerEvaluator<EvalT>(ev);
}
//Planar case:
else {
fm0.template registerEvaluator<EvalT>
(evalUtils.constructGatherCoordinateVectorEvaluator());
}
// fm0.template registerEvaluator<EvalT>
// (evalUtils.constructMapToPhysicalFrameEvaluator(cellType, cubature));
// Shells: 3 coords for 2D topology
// if (spatialDim != modelDim)
if(1)
{
RCP<ParameterList> p = rcp(new ParameterList("Compute Basis Functions"));
// Inputs: X, Y at nodes, Cubature, and Basis
p->set< RCP<Intrepid2::Cubature<PHX::Device> > >("Cubature", cubature);
p->set< RCP<Intrepid2::Basis<PHX::Device, RealType, RealType> > >
("Intrepid2 Basis", intrepidBasis);
p->set<RCP<shards::CellTopology> >("Cell Type", cellType);
// Outputs: BF, weightBF, Grad BF, weighted-Grad BF, all in physical space
p->set<string>("Spherical Coord Name", "Lat-Long");
p->set<std::string>("Lambda Coord Nodal Name", "Lat Nodal");
p->set<std::string>("Theta Coord Nodal Name", "Long Nodal");
p->set<string>("Coordinate Vector Name", "Coord Vec");
p->set<string>("Weights Name", "Weights");
p->set<string>("BF Name", "BF");
p->set<string>("Weighted BF Name", "wBF");
p->set<string>("Gradient BF Name", "Grad BF");
p->set<string>("Weighted Gradient BF Name", "wGrad BF");
p->set<string>("Jacobian Det Name", "Jacobian Det");
p->set<string>("Jacobian Name", "Jacobian");
p->set<string>("Jacobian Inv Name", "Jacobian Inv");
p->set<std::size_t>("spatialDim", spatialDim);
ev = rcp(new Aeras::ComputeBasisFunctions<EvalT,AlbanyTraits>(*p,dl));
fm0.template registerEvaluator<EvalT>(ev);
}
//Planar case:
//IK, 2/11/15: Planar case is obsolete I believe. Should it be removed?
else {
fm0.template registerEvaluator<EvalT>
(evalUtils.constructComputeBasisFunctionsEvaluator(cellType, intrepidBasis, cubature));
}
{
RCP<ParameterList> p = rcp(new ParameterList("SW Compute And Scatter Jacobian"));
p->set< Teuchos::ArrayRCP<string> >("Node Residual Names", dof_names);
p->set<std::string>("Weighted BF Name", "wBF");
p->set<std::string>("Weighted Gradient BF Name", "wGrad BF");
p->set<string>("BF Name", "BF");
p->set<string>("Gradient BF Name", "Grad BF");
p->set<std::string>("Lambda Coord Nodal Name", "Lat Nodal");
p->set<std::string>("Theta Coord Nodal Name", "Long Nodal");
p->set<string>("Scatter Field Name", "SW Compute And Scatter Jacobian");
ev = rcp(new Aeras::SW_ComputeAndScatterJac<EvalT,AlbanyTraits>(*p,dl));
fm0.registerEvaluator<EvalT>(ev);
}
if (fieldManagerChoice == Albany::BUILD_RESID_FM) {
PHX::Tag<typename EvalT::MeshScalarT> res_tag("SW Compute And Scatter Jacobian", dl->dummy);
fm0.requireField<EvalT>(res_tag);
}
else if (fieldManagerChoice == Albany::BUILD_RESPONSE_FM) {
Albany::ResponseUtilities<EvalT, PHAL::AlbanyTraits> respUtils(dl);
return respUtils.constructResponses(fm0, *responseList, Teuchos::null, stateMgr);
}
return Teuchos::null;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm Comm;
#endif
// My MPI process rank.
const int MyPID = Comm.MyPID();
// "/Users/sakashitatatsuya/Downloads/barista_trunk_slepc/sample/hamiltonian_matrix.ip"
std::ifstream ifs(argv[1]);
alps::Parameters params(ifs);
Teuchos::oblackholestream blackHole;
std::ostream& out = (MyPID == 0) ? std::cout : blackHole;
barista::Hamiltonian<> hamiltonian(params);
matrix_type matrix(hamiltonian.dimension(), hamiltonian.dimension());
hamiltonian.fill<double>(matrix);
int m,n;
int N;
m = n = N = hamiltonian.dimension();
//std::cout << matrix << std::endl;
std::ofstream ofs;
if (MyPID==0) {
ofs.open("anasazi_time.txt");
if (!ofs) {
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
}
//Teuchos::ParameterList GaleriList;
using Teuchos::RCP;
using Teuchos::rcp;
typedef Teuchos::ScalarTraits<double> STS;
const double one = STS::one();
const double zero = STS::zero();
// The problem is defined on a 2D grid, global size is nx * nx.
//int nx = N;
//GaleriList.set("n", nx * nx);
//GaleriList.set("nx", nx);
//GaleriList.set("ny", nx);
//Teuchos::RCP<Epetra_Map> Map = Teuchos::rcp( Galeri::CreateMap("Linear", Comm, GaleriList) );
//Teuchos::RCP<Epetra_RowMatrix> A = Teuchos::rcp( Galeri::CreateCrsMatrix("Laplace2D", &*Map, GaleriList) );
// Construct a Map that puts approximately the same number of rows
// of the matrix A on each processor.
Epetra_Map RowMap (N, 0, Comm);
Epetra_Map ColMap (N, 0, Comm);
// Get update list and number of local equations from newly created Map.
const int NumMyRowElements = RowMap.NumMyElements ();
std::vector<int> MyGlobalRowElements (NumMyRowElements);
RowMap.MyGlobalElements (&MyGlobalRowElements[0]);
// Create an Epetra_CrsMatrix using the given row map.
RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Copy, RowMap, n));
// We use info to catch any errors that may have happened during // matrix assembly, and report them globally. We do this so that // the MPI processes won't call FillComplete() unless they all // successfully filled their parts of the matrix.
int info = 0;
try {
//
// Compute coefficients for the discrete integral operator.
//
std::vector<double> Values (n);
std::vector<int> Indices (n);
//const double inv_mp1 = one / (m+1);
//const double inv_np1 = one / (n+1);
int count;
//for (int i = 0; i < n; ++i) {
// Indices[i] = i;
//}
for (int i = 0; i < NumMyRowElements; ++i) {
count =0;
for (int j = 0; j < n; ++j) {
if (matrix(MyGlobalRowElements[i],j)!=0) {
Values[count] = matrix(MyGlobalRowElements[i],j);
Indices[count] = j;
count++;
}
}
info = A->InsertGlobalValues (MyGlobalRowElements[i], count,
&Values[0], &Indices[0]);
// Make sure that the insertion succeeded. Teuchos'
// TEST_FOR_EXCEPTION macro gives a nice error message if the
// thrown exception isn't caught. We'll report this on the
// offending MPI process.
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error, "Failed to insert n="
<< n << " global value" << (n != 1 ? "s" : "")
<< " in row " << MyGlobalRowElements[i]
<< " of the matrix." );
*/
} // for i = 0...
// Call FillComplete on the matrix. Since the matrix isn't square,
// we have to give FillComplete the domain and range maps, which in
// this case are the column resp. row maps.
info = A->FillComplete (ColMap, RowMap);
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error,
"FillComplete failed with INFO = " << info << ".");
*/
info = A->OptimizeStorage();
/*
TEST_FOR_EXCEPTION( info != 0, std::runtime_error,
"OptimizeStorage failed with INFO = " << info << ".");
*/
} catch (std::runtime_error& e) {
// If multiple MPI processes are reporting errors, sometimes
// forming the error message as a string and then writing it to
// the output stream prevents messages from different processes
// from being interleaved.
std::ostringstream os;
os << "*** Error on MPI process " << MyPID << ": " << e.what();
cerr << os.str() << endl;
if (info == 0)
info = -1; // All procs will share info later on.
}
// Variables used for the Block Davidson Method
const int nev = 5;
const int blockSize = 5;
const int numBlocks = 8;
const int maxRestarts = 500;
const double tol = 1.0e-8;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<double, Epetra_MultiVector> MVT;
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
//
//Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(*Map, blockSize) );
Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(ColMap, blockSize) );
ivec->Random();
// Create the eigenproblem.
Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > problem =
Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(A, ivec) );
// Inform the eigenproblem that the operator A is symmetric
problem->setHermitian(true);
// Set the number of eigenvalues requested
problem->setNEV( nev );
// Inform the eigenproblem that you are finishing passing it information
bool boolret = problem->setProblem();
if (boolret != true) {
std::cout<<"Anasazi::BasicEigenproblem::setProblem() returned an error." << std::endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return -1;
}
// Create parameter list to pass into the solver manager
Teuchos::ParameterList anasaziPL;
anasaziPL.set( "Which", "LM" );
anasaziPL.set( "Block Size", blockSize );
anasaziPL.set( "Maximum Iterations", 500 );
anasaziPL.set( "Convergence Tolerance", tol );
anasaziPL.set( "Verbosity", Anasazi::Errors+Anasazi::Warnings+Anasazi::TimingDetails+Anasazi::FinalSummary );
// Create the solver manager
Anasazi::LOBPCGSolMgr<double, MV, OP> anasaziSolver(problem, anasaziPL);
// Solve the problem
double start, end;
MPI_Barrier(MPI_COMM_WORLD);
start = MPI_Wtime();
Anasazi::ReturnType returnCode = anasaziSolver.solve();
MPI_Barrier(MPI_COMM_WORLD);
end = MPI_Wtime();
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<double,MV> sol = problem->getSolution();
std::vector<Anasazi::Value<double> > evals = sol.Evals;
Teuchos::RCP<MV> evecs = sol.Evecs;
// Compute residuals.
std::vector<double> normR(sol.numVecs);
Teuchos::SerialDenseMatrix<int,double> T(sol.numVecs, sol.numVecs);
Epetra_MultiVector tempAevec( ColMap, sol.numVecs );
T.putScalar(0.0);
for (int i=0; i<sol.numVecs; i++) {
T(i,i) = evals[i].realpart;
}
A->Apply( *evecs, tempAevec );
MVT::MvTimesMatAddMv( -1.0, *evecs, T, 1.0, tempAevec );
MVT::MvNorm( tempAevec, normR );
if (MyPID == 0) {
// Print the results
std::cout<<"Solver manager returned " << (returnCode == Anasazi::Converged ? "converged." : "unconverged.") << std::endl;
std::cout<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
std::cout<<std::setw(16)<<"Eigenvalue"
<<std::setw(18)<<"Direct Residual"
<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
for (int i=0; i<sol.numVecs; i++) {
std::cout<<std::setw(16)<<evals[i].realpart
<<std::setw(18)<<normR[i]/evals[i].realpart
<<std::endl;
}
std::cout<<"------------------------------------------------------"<<std::endl;
}
// Print out the map and matrices
//ColMap.Print (out);
//A->Print (cout);
//RowMap.Print (cout);
double time;
int iter;
if (MyPID==0) {
iter = anasaziSolver.getNumIters();
Teuchos::Array<Teuchos::RCP<Teuchos::Time> > timer = anasaziSolver.getTimers();
Teuchos::RCP<Teuchos::Time> _timerSolve = timer[0];
cout << "timerSolve=" << _timerSolve << endl;
time = end - start;
cout << "time=" << time << endl;
ofs << "time=" << time << endl;
cout << "iter=" << iter << endl;
ofs << "iter=" << iter << endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return 0;
}
//
// Test for Tpetra::CrsMatrix::sumIntoGlobalValues(), with nonowned
// rows. This test is like CrsMatrix_NonlocalSumInto.cpp, except that
// it attempts to sum into remote entries that don't exist on the
// process that owns them. Currently, CrsMatrix silently ignores
// these entries. (This is how CrsMatrix implements Import and Export
// when the target matrix has a fixed column Map. Data are
// redistributed between the two row Maps, and "filtered" by the
// target matrix's column Map.) This unit test verifies that behavior
// by ensuring the following:
//
// 1. fillComplete() (actually globalAssemble()) does not throw an
// exception when the incoming entries don't exist on the process
// that owns their rows.
//
// 2. The ignored entries are actually ignored. They must change
// neither the structure nor the values of the matrix.
//
// mfh 16 Dec 2012: The one-template-argument version breaks explicit
// instantiation. Ah well.
//
//TEUCHOS_UNIT_TEST_TEMPLATE_1_DECL( CrsMatrix, NonlocalSumInto_Ignore, CrsMatrixType )
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, NonlocalSumInto_Ignore, LocalOrdinalType, GlobalOrdinalType, ScalarType, NodeType )
{
using Tpetra::createContigMapWithNode;
using Tpetra::createNonContigMapWithNode;
using Tpetra::global_size_t;
using Tpetra::Map;
using Teuchos::Array;
using Teuchos::ArrayView;
using Teuchos::as;
using Teuchos::av_const_cast;
using Teuchos::Comm;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcp_const_cast;
using Teuchos::OrdinalTraits;
using Teuchos::outArg;
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::reduceAll;
using Teuchos::ScalarTraits;
using Teuchos::tuple;
using Teuchos::TypeNameTraits;
using std::endl;
#if 0
// Extract typedefs from the CrsMatrix specialization.
typedef typename CrsMatrixType::scalar_type scalar_type;
typedef typename CrsMatrixType::local_ordinal_type local_ordinal_type;
typedef typename CrsMatrixType::global_ordinal_type global_ordinal_type;
typedef typename CrsMatrixType::node_type node_type;
#endif // 0
typedef ScalarType scalar_type;
typedef LocalOrdinalType local_ordinal_type;
typedef GlobalOrdinalType global_ordinal_type;
typedef NodeType node_type;
// Typedefs derived from the above canonical typedefs.
typedef ScalarTraits<scalar_type> STS;
typedef Map<local_ordinal_type, global_ordinal_type, node_type> map_type;
// Abbreviation typedefs.
typedef scalar_type ST;
typedef local_ordinal_type LO;
typedef global_ordinal_type GO;
typedef node_type NT;
typedef Tpetra::CrsMatrix<ST, LO, GO, NT> CrsMatrixType;
// CrsGraph specialization corresponding to CrsMatrixType (the
// CrsMatrix specialization).
typedef Tpetra::CrsGraph<LO, GO, NT, typename CrsMatrixType::mat_solve_type> crs_graph_type;
////////////////////////////////////////////////////////////////////
// HERE BEGINS THE TEST.
////////////////////////////////////////////////////////////////////
const global_size_t INVALID = OrdinalTraits<global_size_t>::invalid();
// Get the default communicator.
RCP<const Comm<int> > comm = Tpetra::DefaultPlatform::getDefaultPlatform ().getComm ();
const int numProcs = comm->getSize ();
const int myRank = comm->getRank ();
if (myRank == 0) {
out << "Test with " << numProcs << " process" << (numProcs != 1 ? "es" : "") << endl;
}
// This test doesn't make much sense if there is only one MPI
// process. We let it pass trivially in that case.
if (numProcs == 1) {
out << "Number of processes in world is one; test passes trivially." << endl;
return;
}
// Get a Kokkos Node instance. It would be nice if we could pass in
// parameters here, but threads don't matter for this test; it's a
// test for distributed-memory capabilities.
if (myRank == 0) {
out << "Creating Kokkos Node of type " << TypeNameTraits<node_type>::name () << endl;
}
RCP<node_type> node;
{
ParameterList pl; // Kokkos Node types require a PL inout.
node = rcp (new node_type (pl));
}
// Number of rows in the matrix owned by each process.
const LO numLocalRows = 10;
//CrT: 4Feb14: the void trick does not seem to work, I get warnings
// Number of (global) rows and columns in the matrix.
//const GO numGlobalRows = numLocalRows * numProcs;
//const GO numGlobalCols = numGlobalRows;
// Prevent compile warning for unused variable.
// (It's not really "variable" if it's const, but oh well.)
//(void) numGlobalCols;
if (myRank == 0) {
out << "Creating contiguous row Map" << endl;
}
// Create a contiguous row Map, with numLocalRows rows per process.
RCP<const map_type> rowMap = createContigMapWithNode<LO, GO, NT> (INVALID, numLocalRows, comm, node);
// For now, reuse the row Map for the domain and range Maps. Later,
// we might want to test using different domain or range Maps.
RCP<const map_type> domainMap = rowMap;
RCP<const map_type> rangeMap = rowMap;
// Min and max row and column index of this process. Use the row
// Map for the row and column indices, since we're only inserting
// indices into the graph for rows that the calling process owns.
const GO globalMinRow = rowMap->getMinGlobalIndex ();
const GO globalMaxRow = rowMap->getMaxGlobalIndex ();
const GO globalMinCol = domainMap->getMinAllGlobalIndex ();
const GO globalMaxCol = domainMap->getMaxAllGlobalIndex ();
if (myRank == 0) {
out << "Creating graph" << endl;
}
// Create a numGlobalRows by numGlobalCols graph and set its
// structure. Every process sets its diagonal entries (which it
// owns). Unlike in the CrsMatrix_NonlocalSumInto.cpp test, we
// don't set any other entries. As a result, the later calls to
// sumIntoGlobalValues() for nonowned rows should fail.
RCP<const crs_graph_type> graph;
{
// We have a good upper bound for the number of entries per row,
// so use static profile. Leave the upper bound as 2 (just as it
// is in the CrsMatrix_NonlocalSumInto.cpp test) so that there
// would actually be room for the incoming entries from remote
// calls to sumIntoGlobalValues().
RCP<crs_graph_type> nonconstGraph (new crs_graph_type (rowMap, 2, Tpetra::StaticProfile));
TEUCHOS_TEST_FOR_EXCEPTION(globalMinRow >= globalMaxRow, std::logic_error,
"This test only works if globalMinRow < globalMaxRow.");
// Insert all the diagonal entries, and only the diagonal entries
// (unlike in the other test).
for (GO globalRow = globalMinRow; globalRow <= globalMaxRow; ++globalRow) {
nonconstGraph->insertGlobalIndices (globalRow, tuple (globalRow));
}
nonconstGraph->fillComplete (domainMap, rangeMap);
graph = rcp_const_cast<const crs_graph_type> (nonconstGraph);
}
// Test whether the graph has the correct structure.
bool localGraphSuccess = true;
std::ostringstream graphFailMsg;
{
Array<GO> ind (2); // upper bound
for (GO globalRow = globalMinRow; globalRow <= globalMaxRow; ++globalRow) {
size_t numEntries = 0; // output argument of below line.
graph->getGlobalRowCopy (globalRow, ind (), numEntries);
// Revise view based on numEntries.
ArrayView<GO> indView = ind.view (0, numEntries);
// Sort the view.
std::sort (indView.begin (), indView.end ());
if (numEntries != as<size_t> (1)) {
localGraphSuccess = false;
graphFailMsg << "Proc " << myRank << ": globalRow = " << globalRow << ": numEntries = " << numEntries << " != 1" << endl;
}
if (numEntries > 0 && indView[0] != globalRow) {
localGraphSuccess = false;
graphFailMsg << "Proc " << myRank << ": globalRow = " << globalRow << ": indView[0] = " << indView[0] << " != globalRow = " << globalRow << endl;
}
}
}
// Make sure that all processes successfully created the graph.
bool globalGraphSuccess = true;
{
int globalGraphSuccess_int = 1;
reduceAll (*comm, Teuchos::REDUCE_MIN, localGraphSuccess ? 1 : 0, outArg (globalGraphSuccess_int));
globalGraphSuccess = (globalGraphSuccess_int != 0);
}
if (! globalGraphSuccess) {
if (myRank == 0) {
out << "Graph structure not all correct:" << endl << endl;
}
// Print out the failure messages on all processes.
for (int p = 0; p < numProcs; ++p) {
if (p == myRank) {
out << graphFailMsg.str () << endl;
std::flush (out);
}
// Do some barriers to allow output to finish.
comm->barrier ();
comm->barrier ();
comm->barrier ();
}
}
TEUCHOS_TEST_FOR_EXCEPTION(! globalGraphSuccess, std::logic_error, "Graph structure test failed.");
if (myRank == 0) {
out << "Creating matrix" << endl;
}
// Create the matrix, using the above graph.
RCP<CrsMatrixType> matrix (new CrsMatrixType (graph));
if (myRank == 0) {
out << "Setting all matrix entries to 1" << endl;
}
// Set all the owned entries to one. Later we'll set nonlocal
// entries' values in a loop.
matrix->setAllToScalar (STS::one ());
// Attempt to sum into nonowned entries (which nevertheless exist in
// the matrix, just not on this process) using this process' rank.
// The sumIntoGlobalValues() calls will record the data, but the
// globalAssemble() method (called by fillComplete()) will silently
// ignore entries whose columns are not in the column Map. The
// comment at the top of this test explains why this behavior is
// reasonable.
//
// mfh 15,16 Dec 2012: Silently ignoring columns not in the column
// Map has implications for the implementation of
// sumIntoGlobalValues() for nonowned rows. In particular, a
// version of Map's getRemoteIDList() that uses one-sided
// communication could invoke MPI_Get to figure out what the remote
// process owns, without asking it or otherwise requiring
// synchronization. Thus, sumIntoGlobalValues() could throw
// immediately on the calling process, rather than deferring the
// exception to the remote process in globalAssemble(). If we
// switch to that implementation, this unit test must be changed
// accordingly.
if (globalMinRow > rowMap->getMinAllGlobalIndex ()) {
// Attempt to write to the (numLocalRows-1,numLocalCols-1) local entry of the previous process.
matrix->sumIntoGlobalValues (globalMinRow-1, tuple (globalMaxCol), tuple (as<ST> (myRank)));
}
if (globalMaxRow < rowMap->getMaxAllGlobalIndex ()) {
// Attempt to write to the (0,0) local entry of the next process.
matrix->sumIntoGlobalValues (globalMaxRow+1, tuple (globalMinCol), tuple (as<ST> (myRank)));
}
if (myRank == 0) {
out << "Calling fillComplete on the matrix" << endl;
}
TEST_NOTHROW(matrix->fillComplete (domainMap, rangeMap)); // Tpetra::Details::InvalidGlobalIndex<GO>
// mfh 15 Dec 2012: We currently don't make promises about the state
// of the matrix if fillComplete() throws. Later, we might like to
// improve the exception guarantees of fillComplete(). In that
// case, the commented-out code below should be allowed to run.
if (myRank == 0) {
out << "Testing the matrix values" << endl;
}
// Test whether the entries have their correct values.
bool localSuccess = true;
std::ostringstream failMsg;
{
Array<GO> ind (2); // upper bound
Array<ST> val (2); // upper bound
for (GO globalRow = globalMinRow; globalRow <= globalMaxRow; ++globalRow) {
size_t numEntries = 0; // output argument of below line.
matrix->getGlobalRowCopy (globalRow, ind (), val (), numEntries);
// Revise views based on numEntries.
ArrayView<GO> indView = ind.view (0, numEntries);
ArrayView<ST> valView = val.view (0, numEntries);
// Sort the views jointly by column index.
Tpetra::sort2 (indView.begin (), indView.end (), valView.begin ());
if (numEntries != as<size_t> (1)) {
localSuccess = false;
failMsg << "Proc " << myRank << ": globalRow = " << globalRow << ": numEntries = " << numEntries << " != 1" << endl;
}
if (numEntries > 0 && indView[0] != globalRow) {
localSuccess = false;
failMsg << "Proc " << myRank << ": globalRow = " << globalRow << ": indView[0] = " << indView[0] << " != globalRow = " << globalRow << endl;
}
if (numEntries > 0 && valView[0] != STS::one ()) {
localSuccess = false;
failMsg << "Proc " << myRank << ": globalRow = " << globalRow << ": valView[0] = " << valView[0] << " != 1" << endl;
}
}
}
bool globalSuccess = true;
{
int globalSuccess_int = 1;
reduceAll (*comm, Teuchos::REDUCE_MIN, localSuccess ? 1 : 0, outArg (globalSuccess_int));
globalSuccess = (globalSuccess_int != 0);
}
if (! globalSuccess) {
// Print out the failure messages on all processes.
for (int p = 0; p < numProcs; ++p) {
if (p == myRank) {
out << failMsg.str () << endl;
out << "Proc " << myRank << ": localSuccess = " << localSuccess << ", globalSuccess = " << globalSuccess << endl;
// std::flush (out);
}
// Do some barriers to allow output to finish.
comm->barrier ();
comm->barrier ();
comm->barrier ();
}
}
TEST_EQUALITY_CONST(globalSuccess, true);
}
TEUCHOS_UNIT_TEST( Map, ProblematicLookup )
{
using std::cerr;
using std::endl;
Teuchos::oblackholestream blackhole;
RCP<const Teuchos::Comm<int> > comm = Tpetra::DefaultPlatform::getDefaultPlatform().getComm();
const int myRank = comm->getRank();
/**********************************************************************************/
// Map in question:
// -----------------------------
// SRC Map Processor 0: Global IDs = 0 1 2 3 4 5 6
// Processor 1: Global IDs = 9 10 11 12 13 14 15
//
// Lookup of global IDs 7 8 should return IDNotFound
//
if (myRank == 0) {
cerr << "Creating Map" << endl;
}
comm->barrier ();
comm->barrier ();
comm->barrier (); // Just to make sure output finishes.
RCP<const Tpetra::Map<int,int> > map;
if (myRank == 0) {
Array<int> gids( tuple<int>(1) );
map = Tpetra::createNonContigMap<int>( gids().getConst() , comm );
}
else {
Array<int> gids( tuple<int>(3) );
map = Tpetra::createNonContigMap<int>( gids().getConst() , comm );
}
{
std::ostringstream os;
os << "Proc " << myRank << ": created Map" << endl;
cerr << os.str ();
}
{
std::ostringstream os;
os << "Proc " << myRank << ": calling getRemoteIndexList" << endl;
cerr << os.str ();
}
Array<int> nodeIDs( 1 );
Tpetra::LookupStatus lookup = map->getRemoteIndexList( tuple<int>(2), nodeIDs() );
{
std::ostringstream os;
os << "Proc " << myRank << ": getRemoteIndexList done" << endl;
cerr << os.str ();
}
comm->barrier ();
if (myRank == 0) {
cerr << "getRemoteIndexList finished on all processes" << endl;
}
comm->barrier (); // Just to make sure output finishes.
TEST_EQUALITY_CONST( map->isDistributed(), true )
TEST_EQUALITY_CONST( map->isContiguous(), false )
TEST_EQUALITY_CONST( lookup, Tpetra::IDNotPresent )
TEST_COMPARE_ARRAYS( nodeIDs(), tuple<int>(-1) );
}
int main(int argc, char *argv[]) {
typedef double ST;
typedef ScalarTraits<ST> SCT;
typedef SCT::magnitudeType MT;
typedef Tpetra::Operator<ST> OP;
typedef Tpetra::MultiVector<ST> MV;
typedef Tpetra::Vector<ST> VV;
typedef Belos::OperatorTraits<ST,MV,OP> OPT;
typedef Belos::MultiVecTraits<ST,MV> MVT;
GlobalMPISession mpisess(&argc,&argv,&cout);
bool success = false;
bool verbose = false;
try {
const ST one = SCT::one();
int MyPID = 0;
typedef Tpetra::Map<>::node_type Node;
RCP<const Comm<int> > comm = Tpetra::getDefaultComm();
RCP<Node> node; // only for type deduction; null ok
//
// Get test parameters from command-line processor
//
bool proc_verbose = false;
bool debug = false;
int frequency = -1; // how often residuals are printed by solver
int numrhs = 1; // total number of right-hand sides to solve for
int maxiters = -1; // maximum number of iterations for solver to use
std::string filename("bcsstk14.hb");
MT tol = 1.0e-5; // relative residual tolerance
bool precond = false; // use diagonal preconditioner
bool leftPrecond = false; // if preconditioner is used, left or right?
CommandLineProcessor cmdp(false,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("debug","nodebug",&debug,"Run debugging checks.");
cmdp.setOption("frequency",&frequency,"Solvers frequency for printing residuals (#iters).");
cmdp.setOption("tol",&tol,"Relative residual tolerance used by fixed point solver.");
cmdp.setOption("filename",&filename,"Filename for Harwell-Boeing test matrix.");
cmdp.setOption("num-rhs",&numrhs,"Number of right-hand sides to be solved for.");
cmdp.setOption("max-iters",&maxiters,"Maximum number of iterations per linear system (-1 := adapted to problem/block size).");
cmdp.setOption("use-precond","no-precond",&precond,"Use a diagonal preconditioner.");
cmdp.setOption("left","right",&leftPrecond,"Use a left/right preconditioner.");
if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
return -1;
}
if (debug) {
verbose = true;
}
if (!verbose) {
frequency = -1; // reset frequency if test is not verbose
}
MyPID = rank(*comm);
proc_verbose = ( verbose && (MyPID==0) );
if (proc_verbose) {
std::cout << Belos::Belos_Version() << std::endl << std::endl;
}
//
// Get the data from the HB file and build the Map,Matrix
//
RCP<CrsMatrix<ST> > A;
Tpetra::Utils::readHBMatrix(filename,comm,node,A);
RCP<const Tpetra::Map<> > map = A->getDomainMap();
// Create initial vectors
RCP<MV> B, X;
X = rcp( new MV(map,numrhs) );
MVT::MvRandom( *X );
B = rcp( new MV(map,numrhs) );
OPT::Apply( *A, *X, *B );
MVT::MvInit( *X, 0.0 );
// Scale down the rhs
std::vector<ST> norm( MVT::GetNumberVecs(*B));
MVT::MvNorm(*B,norm);
for(int i=0; i< MVT::GetNumberVecs(*B); i++)
norm[i] = 1.0 / norm[i];
MVT::MvScale(*B,norm);
// Drastically bump the diagonal and then rescale so FP can converge
VV diag(A->getRowMap());
A->getLocalDiagCopy(diag);
Teuchos::ArrayRCP<ST> dd=diag.getDataNonConst();
Teuchos::ArrayView<const int> GlobalElements = A->getRowMap()->getNodeElementList();
A->resumeFill();
for(int i=0; i<(int)dd.size(); i++) {
dd[i]=dd[i]*1e4;
A->replaceGlobalValues(GlobalElements[i],
Teuchos::tuple<int>(GlobalElements[i]),
Teuchos::tuple<ST>(dd[i]) );
}
A->fillComplete();
for(int i=0; i<(int)dd.size(); i++)
dd[i] = 1.0/sqrt(dd[i]);
A->leftScale(diag);
A->rightScale(diag);
//
// ********Other information used by block solver***********
// *****************(can be user specified)******************
//
const int NumGlobalElements = B->getGlobalLength();
if (maxiters == -1) {
maxiters = NumGlobalElements- 1; // maximum number of iterations to run
}
//
ParameterList belosList;
belosList.set( "Maximum Iterations", maxiters ); // Maximum number of iterations allowed
belosList.set( "Convergence Tolerance", tol ); // Relative convergence tolerance requested
int verbLevel = Belos::Errors + Belos::Warnings;
if (debug) {
verbLevel += Belos::Debug;
}
if (verbose) {
verbLevel += Belos::TimingDetails + Belos::FinalSummary + Belos::StatusTestDetails;
}
belosList.set( "Verbosity", verbLevel );
if (verbose) {
if (frequency > 0) {
belosList.set( "Output Frequency", frequency );
}
}
//
// Construct an linear problem instance.
//
Belos::LinearProblem<ST,MV,OP> problem( A, X, B );
// diagonal preconditioner
if (precond) {
VV diagonal(A->getRowMap());
A->getLocalDiagCopy(diagonal);
int NumMyElements = diagonal.getMap()->getNodeNumElements();
Teuchos::ArrayView<const int> MyGlobalElements = diagonal.getMap()->getNodeElementList();
Teuchos::ArrayRCP<ST> dd=diagonal.getDataNonConst();
RCP<CrsMatrix<ST> > invDiagMatrix = Teuchos::rcp(new CrsMatrix<ST>(A->getRowMap(), 1, Tpetra::StaticProfile));
for (Teuchos_Ordinal i=0; i<NumMyElements; ++i) {
invDiagMatrix->insertGlobalValues(MyGlobalElements[i],
Teuchos::tuple<int>(MyGlobalElements[i]),
Teuchos::tuple<ST>(SCT::one() / dd[i]) );
}
invDiagMatrix->fillComplete();
if (leftPrecond)
problem.setLeftPrec(invDiagMatrix);
else
problem.setRightPrec(invDiagMatrix);
}
bool set = problem.setProblem();
if (set == false) {
if (proc_verbose)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return -1;
}
//
// *******************************************************************
// *************Start the fixed point iteration***********************
// *******************************************************************
//
Belos::FixedPointSolMgr<ST,MV,OP> solver( rcpFromRef(problem), rcpFromRef(belosList) );
//
// **********Print out information about problem*******************
//
if (proc_verbose) {
std::cout << std::endl << std::endl;
std::cout << "Dimension of matrix: " << NumGlobalElements << std::endl;
std::cout << "Number of right-hand sides: " << numrhs << std::endl;
std::cout << "Max number of fixed point iterations: " << maxiters << std::endl;
std::cout << "Relative residual tolerance: " << tol << std::endl;
std::cout << std::endl;
}
//
// Perform solve
//
Belos::ReturnType ret = solver.solve();
//
// Compute actual residuals.
//
bool badRes = false;
std::vector<MT> actual_resids( numrhs );
std::vector<MT> rhs_norm( numrhs );
MV resid(map, numrhs);
OPT::Apply( *A, *X, resid );
MVT::MvAddMv( -one, resid, one, *B, resid );
MVT::MvNorm( resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
if (proc_verbose) {
std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
}
for ( int i=0; i<numrhs; i++) {
MT actRes = actual_resids[i]/rhs_norm[i];
if (proc_verbose) {
std::cout<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
}
if (actRes > tol) badRes = true;
}
success = (ret==Belos::Converged && !badRes);
if (success) {
if (proc_verbose)
std::cout << "\nEnd Result: TEST PASSED" << std::endl;
} else {
if (proc_verbose)
std::cout << "\nEnd Result: TEST FAILED" << std::endl;
}
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
} // end test_fp_hb.cpp
// calls MPI_Init and MPI_Finalize
int main(int argc,char * argv[])
{
using std::endl;
using Teuchos::RCP;
using Teuchos::rcp_dynamic_cast;
using panzer::StrPureBasisPair;
using panzer::StrPureBasisComp;
Teuchos::GlobalMPISession mpiSession(&argc,&argv);
RCP<Epetra_Comm> Comm = Teuchos::rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
Teuchos::RCP<Teuchos::Comm<int> > comm = Teuchos::rcp(new Teuchos::MpiComm<int>(Teuchos::opaqueWrapper(MPI_COMM_WORLD)));
Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout));
out.setOutputToRootOnly(0);
out.setShowProcRank(true);
// Build command line processor
////////////////////////////////////////////////////
bool useTpetra = false;
Teuchos::CommandLineProcessor clp;
clp.setOption("use-tpetra","use-epetra",&useTpetra);
// parse commandline argument
TEUCHOS_ASSERT(clp.parse(argc,argv)==Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL);
// variable declarations
////////////////////////////////////////////////////
// factory definitions
Teuchos::RCP<Example::EquationSetFactory> eqset_factory =
Teuchos::rcp(new Example::EquationSetFactory); // where poison equation is defined
Example::BCStrategyFactory bc_factory; // where boundary conditions are defined
panzer_stk_classic::SquareQuadMeshFactory mesh_factory;
// other declarations
const std::size_t workset_size = 2*2;
// construction of uncommitted (no elements) mesh
////////////////////////////////////////////////////////
// set mesh factory parameters
RCP<Teuchos::ParameterList> pl = rcp(new Teuchos::ParameterList);
pl->set("X Blocks",1);
pl->set("Y Blocks",1);
pl->set("X Elements",20);
pl->set("Y Elements",20);
mesh_factory.setParameterList(pl);
RCP<panzer_stk_classic::STK_Interface> mesh = mesh_factory.buildUncommitedMesh(MPI_COMM_WORLD);
// construct input physics and physics block
////////////////////////////////////////////////////////
Teuchos::RCP<Teuchos::ParameterList> ipb = Teuchos::parameterList("Physics Blocks");
std::vector<panzer::BC> bcs;
std::vector<RCP<panzer::PhysicsBlock> > physicsBlocks;
{
bool build_transient_support = false;
testInitialization(ipb, bcs);
const panzer::CellData volume_cell_data(workset_size, mesh->getCellTopology("eblock-0_0"));
// GobalData sets ostream and parameter interface to physics
Teuchos::RCP<panzer::GlobalData> gd = panzer::createGlobalData();
// Can be overridden by the equation set
int default_integration_order = 1;
// the physics block nows how to build and register evaluator with the field manager
RCP<panzer::PhysicsBlock> pb
= rcp(new panzer::PhysicsBlock(ipb, "eblock-0_0",
default_integration_order,
volume_cell_data,
eqset_factory,
gd,
build_transient_support));
// we can have more than one physics block, one per element block
physicsBlocks.push_back(pb);
}
// finish building mesh, set required field variables and mesh bulk data
////////////////////////////////////////////////////////////////////////
{
RCP<panzer::PhysicsBlock> pb = physicsBlocks[0]; // we are assuming only one physics block
const std::vector<StrPureBasisPair> & blockFields = pb->getProvidedDOFs();
// insert all fields into a set
std::set<StrPureBasisPair,StrPureBasisComp> fieldNames;
fieldNames.insert(blockFields.begin(),blockFields.end());
// build string for modifiying vectors
std::vector<std::string> dimenStr(3);
dimenStr[0] = "X"; dimenStr[1] = "Y"; dimenStr[2] = "Z";
// add basis to DOF manager: block specific
std::set<StrPureBasisPair,StrPureBasisComp>::const_iterator fieldItr;
for (fieldItr=fieldNames.begin();fieldItr!=fieldNames.end();++fieldItr) {
Teuchos::RCP<const panzer::PureBasis> basis = fieldItr->second;
if(basis->getElementSpace()==panzer::PureBasis::HGRAD)
mesh->addSolutionField(fieldItr->first,pb->elementBlockID());
else if(basis->getElementSpace()==panzer::PureBasis::HCURL) {
for(int i=0;i<basis->dimension();i++)
mesh->addCellField(fieldItr->first+dimenStr[i],pb->elementBlockID());
}
}
mesh_factory.completeMeshConstruction(*mesh,MPI_COMM_WORLD);
}
// build DOF Manager and linear object factory
/////////////////////////////////////////////////////////////
RCP<panzer::UniqueGlobalIndexerBase> dofManager;
Teuchos::RCP<panzer::LinearObjFactory<panzer::Traits> > linObjFactory;
// build the connection manager
if(!useTpetra) {
const Teuchos::RCP<panzer::ConnManager<int,int> > conn_manager = Teuchos::rcp(new panzer_stk_classic::STKConnManager<int>(mesh));
panzer::DOFManagerFactory<int,int> globalIndexerFactory;
RCP<panzer::UniqueGlobalIndexer<int,int> > dofManager_int
= globalIndexerFactory.buildUniqueGlobalIndexer(Teuchos::opaqueWrapper(MPI_COMM_WORLD),physicsBlocks,conn_manager);
dofManager = dofManager_int;
// construct some linear algebra object, build object to pass to evaluators
linObjFactory = Teuchos::rcp(new panzer::EpetraLinearObjFactory<panzer::Traits,int>(Comm.getConst(),dofManager_int));
}
else {
const Teuchos::RCP<panzer::ConnManager<int,panzer::Ordinal64> > conn_manager = Teuchos::rcp(new panzer_stk_classic::STKConnManager<panzer::Ordinal64>(mesh));
panzer::DOFManagerFactory<int,panzer::Ordinal64> globalIndexerFactory;
RCP<panzer::UniqueGlobalIndexer<int,panzer::Ordinal64> > dofManager_long
= globalIndexerFactory.buildUniqueGlobalIndexer(Teuchos::opaqueWrapper(MPI_COMM_WORLD),physicsBlocks,conn_manager);
dofManager = dofManager_long;
// construct some linear algebra object, build object to pass to evaluators
linObjFactory = Teuchos::rcp(new panzer::TpetraLinearObjFactory<panzer::Traits,double,int,panzer::Ordinal64>(comm,dofManager_long));
}
// build worksets
////////////////////////////////////////////////////////
Teuchos::RCP<panzer_stk_classic::WorksetFactory> wkstFactory
= Teuchos::rcp(new panzer_stk_classic::WorksetFactory(mesh)); // build STK workset factory
Teuchos::RCP<panzer::WorksetContainer> wkstContainer // attach it to a workset container (uses lazy evaluation)
= Teuchos::rcp(new panzer::WorksetContainer(wkstFactory,physicsBlocks,workset_size));
wkstContainer->setGlobalIndexer(dofManager);
// Setup STK response library for writing out the solution fields
////////////////////////////////////////////////////////////////////////
Teuchos::RCP<panzer::ResponseLibrary<panzer::Traits> > stkIOResponseLibrary
= Teuchos::rcp(new panzer::ResponseLibrary<panzer::Traits>(wkstContainer,dofManager,linObjFactory));
{
// get a vector of all the element blocks
std::vector<std::string> eBlocks;
{
// get all element blocks and add them to the list
std::vector<std::string> eBlockNames;
mesh->getElementBlockNames(eBlockNames);
for(std::size_t i=0;i<eBlockNames.size();i++)
eBlocks.push_back(eBlockNames[i]);
}
panzer_stk_classic::RespFactorySolnWriter_Builder builder;
builder.mesh = mesh;
stkIOResponseLibrary->addResponse("Main Field Output",eBlocks,builder);
}
// setup closure model
/////////////////////////////////////////////////////////////
// Add in the application specific closure model factory
panzer::ClosureModelFactory_TemplateManager<panzer::Traits> cm_factory;
Example::ClosureModelFactory_TemplateBuilder cm_builder;
cm_factory.buildObjects(cm_builder);
Teuchos::ParameterList closure_models("Closure Models");
closure_models.sublist("solid").sublist("SOURCE_EFIELD").set<std::string>("Type","SIMPLE SOURCE"); // a constant source
// SOURCE_EFIELD field is required by the CurlLaplacianEquationSet
Teuchos::ParameterList user_data("User Data"); // user data can be empty here
// setup field manager builder
/////////////////////////////////////////////////////////////
Teuchos::RCP<panzer::FieldManagerBuilder> fmb =
Teuchos::rcp(new panzer::FieldManagerBuilder);
fmb->setWorksetContainer(wkstContainer);
fmb->setupVolumeFieldManagers(physicsBlocks,cm_factory,closure_models,*linObjFactory,user_data);
fmb->setupBCFieldManagers(bcs,physicsBlocks,*eqset_factory,cm_factory,bc_factory,closure_models,
*linObjFactory,user_data);
// setup assembly engine
/////////////////////////////////////////////////////////////
// build assembly engine: The key piece that brings together everything and
// drives and controls the assembly process. Just add
// matrices and vectors
panzer::AssemblyEngine_TemplateManager<panzer::Traits> ae_tm;
panzer::AssemblyEngine_TemplateBuilder builder(fmb,linObjFactory);
ae_tm.buildObjects(builder);
// Finalize construcition of STK writer response library
/////////////////////////////////////////////////////////////
{
user_data.set<int>("Workset Size",workset_size);
stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks,
cm_factory,
closure_models,
user_data);
}
// assemble linear system
/////////////////////////////////////////////////////////////
// build linear algebra objects: Ghost is for parallel assembly, it contains
// local element contributions summed, the global IDs
// are not unique. The non-ghosted or "global"
// container will contain the sum over all processors
// of the ghosted objects. The global indices are unique.
RCP<panzer::LinearObjContainer> ghostCont = linObjFactory->buildGhostedLinearObjContainer();
RCP<panzer::LinearObjContainer> container = linObjFactory->buildLinearObjContainer();
linObjFactory->initializeGhostedContainer(panzer::LinearObjContainer::X |
panzer::LinearObjContainer::F |
panzer::LinearObjContainer::Mat,*ghostCont);
linObjFactory->initializeContainer(panzer::LinearObjContainer::X |
panzer::LinearObjContainer::F |
panzer::LinearObjContainer::Mat,*container);
ghostCont->initialize();
container->initialize();
// Actually evaluate
/////////////////////////////////////////////////////////////
panzer::AssemblyEngineInArgs input(ghostCont,container);
input.alpha = 0;
input.beta = 1;
// evaluate physics: This does both the Jacobian and residual at once
ae_tm.getAsObject<panzer::Traits::Jacobian>()->evaluate(input);
// solve linear system
/////////////////////////////////////////////////////////////
if(useTpetra)
solveTpetraSystem(*container);
else
solveEpetraSystem(*container);
// output data (optional)
/////////////////////////////////////////////////////////////
// write out solution
if(true) {
// fill STK mesh objects
Teuchos::RCP<panzer::ResponseBase> resp = stkIOResponseLibrary->getResponse<panzer::Traits::Residual>("Main Field Output");
panzer::AssemblyEngineInArgs respInput(ghostCont,container);
respInput.alpha = 0;
respInput.beta = 1;
stkIOResponseLibrary->addResponsesToInArgs<panzer::Traits::Residual>(respInput);
stkIOResponseLibrary->evaluate<panzer::Traits::Residual>(respInput);
// write to exodus
mesh->writeToExodus("output.exo");
}
// all done!
/////////////////////////////////////////////////////////////
if(useTpetra)
std::cout << "ALL PASSED: Tpetra" << std::endl;
else
std::cout << "ALL PASSED: Epetra" << std::endl;
return 0;
}