void DefaultIdentityLinearOp<Scalar>::applyImpl(
const EOpTransp M_trans,
const MultiVectorBase<Scalar> &X,
const Ptr<MultiVectorBase<Scalar> > &Y,
const Scalar alpha,
const Scalar beta
) const
{
using Teuchos::tuple;
using Teuchos::ptrFromRef;
#ifdef TEUCHOS_DEBUG
THYRA_ASSERT_LINEAR_OP_MULTIVEC_APPLY_SPACES(
"DefaultIdentityLinearOp<Scalar>::apply(...)", *this, M_trans, X, &*Y
);
#endif // TEUCHOS_DEBUG
Thyra::linear_combination<Scalar>(
tuple<Scalar>(alpha)(),
tuple<Ptr<const MultiVectorBase<Scalar> > >(ptrFromRef(X))(),
beta, Y
);
}
void panzer::GatherTangent_BlockedTpetra<EvalT, TRAITS,S,LO,GO,NodeT>::
evaluateFields(typename TRAITS::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
// If blockedContainer_ was not initialized, then no global evaluation data
// container was set, in which case this evaluator becomes a no-op
if (blockedContainer_ == Teuchos::null)
return;
Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout));
out.setShowProcRank(true);
out.setOutputToRootOnly(-1);
std::vector<std::pair<int,GO> > GIDs;
std::vector<LO> LIDs;
// for convenience pull out some objects from workset
std::string blockId = this->wda(workset).block_id;
const std::vector<std::size_t> & localCellIds = this->wda(workset).cell_local_ids;
Teuchos::RCP<ProductVectorBase<double> > x;
if (useTimeDerivativeSolutionVector_)
x = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_dxdt());
else
x = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_x());
// gather operation for each cell in workset
for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
LO cellLocalId = localCellIds[worksetCellIndex];
gidIndexer_->getElementGIDs(cellLocalId,GIDs,blockId);
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0;i<GIDs.size();i++) {
// used for doing local ID lookups
RCP<const MapType> x_map = blockedContainer_->getMapForBlock(GIDs[i].first);
LIDs[i] = x_map->getLocalElement(GIDs[i].second);
}
// loop over the fields to be gathered
Teuchos::ArrayRCP<const double> local_x;
for (std::size_t fieldIndex=0; fieldIndex<gatherFields_.size();fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int indexerId = gidIndexer_->getFieldBlock(fieldNum);
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_x = rcp_dynamic_cast<SpmdVectorBase<double> >(x->getNonconstVectorBlock(indexerId));
block_x->getLocalData(ptrFromRef(local_x));
const std::vector<int> & elmtOffset = gidIndexer_->getGIDFieldOffsets(blockId,fieldNum);
// loop over basis functions and fill the fields
for(std::size_t basis=0;basis<elmtOffset.size();basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
(gatherFields_[fieldIndex])(worksetCellIndex,basis) = local_x[lid];
}
}
}
}
void panzer::ScatterResidual_BlockedEpetra<panzer::Traits::Jacobian, Traits,LO,GO>::
evaluateFields(typename Traits::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
using Thyra::BlockedLinearOpBase;
typedef BlockedEpetraLinearObjContainer BLOC;
std::vector<std::pair<int,GO> > GIDs;
std::vector<LO> LIDs;
std::vector<double> jacRow;
// for convenience pull out some objects from workset
std::string blockId = workset.block_id;
const std::vector<std::size_t> & localCellIds = workset.cell_local_ids;
RCP<const BLOC> blockedContainer = blockedContainer_;
RCP<ProductVectorBase<double> > r = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer->get_f());
Teuchos::RCP<BlockedLinearOpBase<double> > Jac = rcp_dynamic_cast<BlockedLinearOpBase<double> >(blockedContainer->get_A());
int numFieldBlocks = globalIndexer_->getNumFieldBlocks();
std::vector<int> blockOffsets(numFieldBlocks+1); // number of fields, plus a sentinnel
for(int blk=0;blk<numFieldBlocks;blk++) {
int blockOffset = globalIndexer_->getBlockGIDOffset(blockId,blk);
blockOffsets[blk] = blockOffset;
}
boost::unordered_map<std::pair<int,int>,Teuchos::RCP<Epetra_CrsMatrix> > jacEpetraBlocks;
// NOTE: A reordering of these loops will likely improve performance
// The "getGIDFieldOffsets" may be expensive. However the
// "getElementGIDs" can be cheaper. However the lookup for LIDs
// may be more expensive!
// scatter operation for each cell in workset
for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
std::size_t cellLocalId = localCellIds[worksetCellIndex];
globalIndexer_->getElementGIDs(cellLocalId,GIDs,blockId);
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0;i<GIDs.size();i++) {
// used for doing local ID lookups
RCP<const Epetra_Map> r_map = blockedContainer->getMapForBlock(GIDs[i].first);
LIDs[i] = r_map->LID(GIDs[i].second);
}
// loop over each field to be scattered
Teuchos::ArrayRCP<double> local_r;
for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int blockRowIndex = globalIndexer_->getFieldBlock(fieldNum);
// grab local data for inputing
if(r!=Teuchos::null) {
RCP<SpmdVectorBase<double> > block_r = rcp_dynamic_cast<SpmdVectorBase<double> >(r->getNonconstVectorBlock(blockRowIndex));
block_r->getNonconstLocalData(ptrFromRef(local_r));
}
const std::vector<int> & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum);
// loop over the basis functions (currently they are nodes)
for(std::size_t rowBasisNum = 0; rowBasisNum < elmtOffset.size(); rowBasisNum++) {
const ScalarT & scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,rowBasisNum);
int rowOffset = elmtOffset[rowBasisNum];
int r_lid = LIDs[rowOffset];
// Sum residual
if(local_r!=Teuchos::null)
local_r[r_lid] += (scatterField.val());
blockOffsets[numFieldBlocks] = scatterField.size(); // add the sentinel
// loop over the sensitivity indices: all DOFs on a cell
jacRow.resize(scatterField.size());
for(int sensIndex=0;sensIndex<scatterField.size();++sensIndex) {
jacRow[sensIndex] = scatterField.fastAccessDx(sensIndex);
}
for(int blockColIndex=0;blockColIndex<numFieldBlocks;blockColIndex++) {
int start = blockOffsets[blockColIndex];
int end = blockOffsets[blockColIndex+1];
if(end-start<=0)
continue;
// check hash table for jacobian sub block
std::pair<int,int> blockIndex = std::make_pair(blockRowIndex,blockColIndex);
Teuchos::RCP<Epetra_CrsMatrix> subJac = jacEpetraBlocks[blockIndex];
// if you didn't find one before, add it to the hash table
if(subJac==Teuchos::null) {
Teuchos::RCP<Thyra::LinearOpBase<double> > tOp = Jac->getNonconstBlock(blockIndex.first,blockIndex.second);
// block operator is null, don't do anything (it is excluded)
if(Teuchos::is_null(tOp))
continue;
Teuchos::RCP<Epetra_Operator> eOp = Thyra::get_Epetra_Operator(*tOp);
subJac = rcp_dynamic_cast<Epetra_CrsMatrix>(eOp,true);
jacEpetraBlocks[blockIndex] = subJac;
}
// Sum Jacobian
int err = subJac->SumIntoMyValues(r_lid, end-start, &jacRow[start],&LIDs[start]);
if(err!=0) {
RCP<const Epetra_Map> rr = blockedContainer->getMapForBlock(GIDs[start].first);
bool sameColMap = subJac->ColMap().SameAs(*rr);
std::stringstream ss;
ss << "Failed inserting row: " << GIDs[rowOffset].second << " (" << r_lid << "): ";
for(int i=start;i<end;i++)
ss << GIDs[i].second << " (" << LIDs[i] << ") ";
ss << std::endl;
ss << "Into block " << blockRowIndex << ", " << blockColIndex << std::endl;
ss << "scatter field = ";
scatterFields_[fieldIndex].print(ss);
ss << std::endl;
ss << "Same map = " << (sameColMap ? "true" : "false") << std::endl;
TEUCHOS_TEST_FOR_EXCEPTION(err!=0,std::runtime_error,ss.str());
}
}
} // end rowBasisNum
} // end fieldIndex
}
}
void panzer::ScatterResidual_BlockedEpetra<panzer::Traits::Residual, Traits,LO,GO>::
evaluateFields(typename Traits::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
typedef BlockedEpetraLinearObjContainer BLOC;
std::vector<std::pair<int,GO> > GIDs;
std::vector<int> LIDs;
// for convenience pull out some objects from workset
std::string blockId = workset.block_id;
const std::vector<std::size_t> & localCellIds = workset.cell_local_ids;
RCP<const BLOC> blockedContainer = blockedContainer_;
RCP<ProductVectorBase<double> > r = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer->get_f(),true);
// NOTE: A reordering of these loops will likely improve performance
// The "getGIDFieldOffsets may be expensive. However the
// "getElementGIDs" can be cheaper. However the lookup for LIDs
// may be more expensive!
// scatter operation for each cell in workset
for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
std::size_t cellLocalId = localCellIds[worksetCellIndex];
globalIndexer_->getElementGIDs(cellLocalId,GIDs,blockId);
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0;i<GIDs.size();i++) {
// used for doing local ID lookups
RCP<const Epetra_Map> r_map = blockedContainer->getMapForBlock(GIDs[i].first);
LIDs[i] = r_map->LID(GIDs[i].second);
}
// loop over each field to be scattered
Teuchos::ArrayRCP<double> local_r;
for (std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int indexerId = globalIndexer_->getFieldBlock(fieldNum);
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_r = rcp_dynamic_cast<SpmdVectorBase<double> >(r->getNonconstVectorBlock(indexerId));
block_r->getNonconstLocalData(ptrFromRef(local_r));
const std::vector<int> & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum);
// loop over basis functions
for(std::size_t basis=0;basis<elmtOffset.size();basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
local_r[lid] += (scatterFields_[fieldIndex])(worksetCellIndex,basis);
}
}
}
}
void panzer::GatherSolution_BlockedTpetra<panzer::Traits::Jacobian, TRAITS,S,LO,GO,NodeT>::
evaluateFields(typename TRAITS::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
std::vector<std::pair<int,GO> > GIDs;
std::vector<LO> LIDs;
// for convenience pull out some objects from workset
std::string blockId = this->wda(workset).block_id;
const std::vector<std::size_t> & localCellIds = this->wda(workset).cell_local_ids;
double seed_value = 0.0;
Teuchos::RCP<ProductVectorBase<double> > x;
if (useTimeDerivativeSolutionVector_) {
x = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_dxdt());
seed_value = workset.alpha;
}
else {
x = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_x());
seed_value = workset.beta;
}
// turn off sensitivies: this may be faster if we don't expand the term
// but I suspect not because anywhere it is used the full complement of
// sensitivies will be needed anyway.
if(disableSensitivities_)
seed_value = 0.0;
// NOTE: A reordering of these loops will likely improve performance
// The "getGIDFieldOffsets may be expensive. However the
// "getElementGIDs" can be cheaper. However the lookup for LIDs
// may be more expensive!
// gather operation for each cell in workset
for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
LO cellLocalId = localCellIds[worksetCellIndex];
gidIndexer_->getElementGIDs(cellLocalId,GIDs,blockId);
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0;i<GIDs.size();i++) {
// used for doing local ID lookups
RCP<const MapType> x_map = blockedContainer_->getMapForBlock(GIDs[i].first);
LIDs[i] = x_map->getLocalElement(GIDs[i].second);
}
// loop over the fields to be gathered
Teuchos::ArrayRCP<const double> local_x;
for(std::size_t fieldIndex=0;
fieldIndex<gatherFields_.size();fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int indexerId = gidIndexer_->getFieldBlock(fieldNum);
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_x = rcp_dynamic_cast<SpmdVectorBase<double> >(x->getNonconstVectorBlock(indexerId));
block_x->getLocalData(ptrFromRef(local_x));
const std::vector<int> & elmtOffset = gidIndexer_->getGIDFieldOffsets(blockId,fieldNum);
// loop over basis functions and fill the fields
for(std::size_t basis=0;basis<elmtOffset.size();basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
// set the value and seed the FAD object
(gatherFields_[fieldIndex])(worksetCellIndex,basis) = ScalarT(GIDs.size(), local_x[lid]);
(gatherFields_[fieldIndex])(worksetCellIndex,basis).fastAccessDx(offset) = seed_value;
}
}
}
}
void panzer::ScatterDirichletResidual_BlockedTpetra<panzer::Traits::Jacobian, TRAITS,LO,GO,NodeT>::
evaluateFields(typename TRAITS::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
using Thyra::BlockedLinearOpBase;
std::vector<std::pair<int,GO> > GIDs;
std::vector<LO> LIDs;
// for convenience pull out some objects from workset
std::string blockId = this->wda(workset).block_id;
const std::vector<std::size_t> & localCellIds = this->wda(workset).cell_local_ids;
RCP<ProductVectorBase<double> > r = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_f());
Teuchos::RCP<BlockedLinearOpBase<double> > Jac = rcp_dynamic_cast<BlockedLinearOpBase<double> >(blockedContainer_->get_A());
int numFieldBlocks = globalIndexer_->getNumFieldBlocks();
std::vector<int> blockOffsets(numFieldBlocks+1); // number of fields, plus a sentinnel
for(int blk=0; blk<numFieldBlocks; blk++) {
int blockOffset = globalIndexer_->getBlockGIDOffset(blockId,blk);
blockOffsets[blk] = blockOffset;
}
std::unordered_map<std::pair<int,int>,Teuchos::RCP<CrsMatrixType>,panzer::pair_hash> jacTpetraBlocks;
// NOTE: A reordering of these loops will likely improve performance
// The "getGIDFieldOffsets may be expensive. However the
// "getElementGIDs" can be cheaper. However the lookup for LIDs
// may be more expensive!
// scatter operation for each cell in workset
for(std::size_t worksetCellIndex=0; worksetCellIndex<localCellIds.size(); ++worksetCellIndex) {
std::size_t cellLocalId = localCellIds[worksetCellIndex];
globalIndexer_->getElementGIDs(cellLocalId,GIDs);
blockOffsets[numFieldBlocks] = GIDs.size();
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0; i<GIDs.size(); i++) {
// used for doing local ID lookups
RCP<const MapType> r_map = blockedContainer_->getMapForBlock(GIDs[i].first);
LIDs[i] = r_map->getLocalElement(GIDs[i].second);
}
// loop over each field to be scattered
Teuchos::ArrayRCP<double> local_r, local_dc;
for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int blockRowIndex = globalIndexer_->getFieldBlock(fieldNum);
RCP<SpmdVectorBase<double> > dc = rcp_dynamic_cast<SpmdVectorBase<double> >(dirichletCounter_->getNonconstVectorBlock(blockRowIndex));
dc->getNonconstLocalData(ptrFromRef(local_dc));
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_r = rcp_dynamic_cast<SpmdVectorBase<double> >(r->getNonconstVectorBlock(blockRowIndex));
block_r->getNonconstLocalData(ptrFromRef(local_r));
// this call "should" get the right ordering according to the Intrepid basis
const std::pair<std::vector<int>,std::vector<int> > & indicePair
= globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_);
const std::vector<int> & elmtOffset = indicePair.first;
const std::vector<int> & basisIdMap = indicePair.second;
// loop over basis functions
for(std::size_t basis=0; basis<elmtOffset.size(); basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
if(lid<0) // not on this processor
continue;
int basisId = basisIdMap[basis];
if (checkApplyBC_)
if (!applyBC_[fieldIndex](worksetCellIndex,basisId))
continue;
// zero out matrix row
for(int blockColIndex=0; blockColIndex<numFieldBlocks; blockColIndex++) {
int start = blockOffsets[blockColIndex];
int end = blockOffsets[blockColIndex+1];
if(end-start<=0)
continue;
// check hash table for jacobian sub block
std::pair<int,int> blockIndex = std::make_pair(blockRowIndex,blockColIndex);
Teuchos::RCP<CrsMatrixType> subJac = jacTpetraBlocks[blockIndex];
// if you didn't find one before, add it to the hash table
if(subJac==Teuchos::null) {
Teuchos::RCP<Thyra::LinearOpBase<double> > tOp = Jac->getNonconstBlock(blockIndex.first,blockIndex.second);
// block operator is null, don't do anything (it is excluded)
if(Teuchos::is_null(tOp))
continue;
Teuchos::RCP<OperatorType> tpetra_Op = rcp_dynamic_cast<ThyraLinearOp>(tOp)->getTpetraOperator();
subJac = rcp_dynamic_cast<CrsMatrixType>(tpetra_Op,true);
jacTpetraBlocks[blockIndex] = subJac;
}
std::size_t sz = subJac->getNumEntriesInLocalRow(lid);
std::size_t numEntries = 0;
Teuchos::Array<LO> rowIndices(sz);
Teuchos::Array<double> rowValues(sz);
subJac->getLocalRowCopy(lid,rowIndices,rowValues,numEntries);
for(std::size_t i=0; i<numEntries; i++)
rowValues[i] = 0.0;
subJac->replaceLocalValues(lid,rowIndices,rowValues);
}
const ScalarT scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,basisId);
local_r[lid] = scatterField.val();
local_dc[lid] = 1.0; // mark row as dirichlet
// loop over the sensitivity indices: all DOFs on a cell
std::vector<double> jacRow(scatterField.size(),0.0);
for(int sensIndex=0; sensIndex<scatterField.size(); ++sensIndex)
jacRow[sensIndex] = scatterField.fastAccessDx(sensIndex);
TEUCHOS_ASSERT(jacRow.size()==GIDs.size());
for(int blockColIndex=0; blockColIndex<numFieldBlocks; blockColIndex++) {
int start = blockOffsets[blockColIndex];
int end = blockOffsets[blockColIndex+1];
if(end-start<=0)
continue;
// check hash table for jacobian sub block
std::pair<int,int> blockIndex = std::make_pair(blockRowIndex,blockColIndex);
Teuchos::RCP<CrsMatrixType> subJac = jacTpetraBlocks[blockIndex];
// if you didn't find one before, add it to the hash table
if(subJac==Teuchos::null) {
Teuchos::RCP<Thyra::LinearOpBase<double> > tOp = Jac->getNonconstBlock(blockIndex.first,blockIndex.second);
// block operator is null, don't do anything (it is excluded)
if(Teuchos::is_null(tOp))
continue;
Teuchos::RCP<OperatorType> tpetra_Op = rcp_dynamic_cast<ThyraLinearOp>(tOp)->getTpetraOperator();
subJac = rcp_dynamic_cast<CrsMatrixType>(tpetra_Op,true);
jacTpetraBlocks[blockIndex] = subJac;
}
// Sum Jacobian
subJac->replaceLocalValues(lid, Teuchos::arrayViewFromVector(LIDs).view(start,end-start),
Teuchos::arrayViewFromVector(jacRow).view(start,end-start));
}
}
}
}
}
void panzer::ScatterDirichletResidual_BlockedTpetra<panzer::Traits::Residual, TRAITS,LO,GO,NodeT>::
evaluateFields(typename TRAITS::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout));
out.setShowProcRank(true);
out.setOutputToRootOnly(-1);
std::vector<std::pair<int,GO> > GIDs;
std::vector<LO> LIDs;
// for convenience pull out some objects from workset
std::string blockId = this->wda(workset).block_id;
const std::vector<std::size_t> & localCellIds = this->wda(workset).cell_local_ids;
RCP<ProductVectorBase<double> > r = (!scatterIC_) ?
rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_f(),true) :
rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_x(),true);
// NOTE: A reordering of these loops will likely improve performance
// The "getGIDFieldOffsets may be expensive. However the
// "getElementGIDs" can be cheaper. However the lookup for LIDs
// may be more expensive!
// scatter operation for each cell in workset
for(std::size_t worksetCellIndex=0; worksetCellIndex<localCellIds.size(); ++worksetCellIndex) {
std::size_t cellLocalId = localCellIds[worksetCellIndex];
globalIndexer_->getElementGIDs(cellLocalId,GIDs);
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0; i<GIDs.size(); i++) {
// used for doing local ID lookups
RCP<const MapType> r_map = blockedContainer_->getMapForBlock(GIDs[i].first);
LIDs[i] = r_map->getLocalElement(GIDs[i].second);
}
// loop over each field to be scattered
Teuchos::ArrayRCP<double> local_r;
Teuchos::ArrayRCP<double> local_dc;
for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int indexerId = globalIndexer_->getFieldBlock(fieldNum);
RCP<SpmdVectorBase<double> > dc = rcp_dynamic_cast<SpmdVectorBase<double> >(dirichletCounter_->getNonconstVectorBlock(indexerId));
dc->getNonconstLocalData(ptrFromRef(local_dc));
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_r = rcp_dynamic_cast<SpmdVectorBase<double> >(r->getNonconstVectorBlock(indexerId));
block_r->getNonconstLocalData(ptrFromRef(local_r));
if (!scatterIC_) {
// this call "should" get the right ordering according to the Intrepid basis
const std::pair<std::vector<int>,std::vector<int> > & indicePair
= globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_);
const std::vector<int> & elmtOffset = indicePair.first;
const std::vector<int> & basisIdMap = indicePair.second;
// loop over basis functions
for(std::size_t basis=0; basis<elmtOffset.size(); basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
if(lid<0) // not on this processor!
continue;
int basisId = basisIdMap[basis];
if (checkApplyBC_)
if (!applyBC_[fieldIndex](worksetCellIndex,basisId))
continue;
local_r[lid] = (scatterFields_[fieldIndex])(worksetCellIndex,basisId);
// record that you set a dirichlet condition
local_dc[lid] = 1.0;
}
} else {
// this call "should" get the right ordering according to the Intrepid basis
const std::vector<int> & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum);
// loop over basis functions
for(std::size_t basis=0; basis<elmtOffset.size(); basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
if(lid<0) // not on this processor!
continue;
local_r[lid] = (scatterFields_[fieldIndex])(worksetCellIndex,basis);
// record that you set a dirichlet condition
local_dc[lid] = 1.0;
}
}
}
}
}
void panzer::GatherSolution_BlockedEpetra<panzer::Traits::Residual, TRAITS,LO,GO>::
evaluateFields(typename TRAITS::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
typedef BlockedEpetraLinearObjContainer BLOC;
Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout));
out.setShowProcRank(true);
out.setOutputToRootOnly(-1);
std::vector<std::pair<int,GO> > GIDs;
std::vector<int> LIDs;
// for convenience pull out some objects from workset
std::string blockId = this->wda(workset).block_id;
const std::vector<std::size_t> & localCellIds = this->wda(workset).cell_local_ids;
Teuchos::RCP<ProductVectorBase<double> > x;
if (useTimeDerivativeSolutionVector_)
x = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_dxdt());
else
x = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer_->get_x());
// gather operation for each cell in workset
for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
LO cellLocalId = localCellIds[worksetCellIndex];
gidIndexer_->getElementGIDs(cellLocalId,GIDs,blockId);
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0;i<GIDs.size();i++) {
// used for doing local ID lookups
RCP<const Epetra_Map> x_map = blockedContainer_->getMapForBlock(GIDs[i].first);
LIDs[i] = x_map->LID(GIDs[i].second);
// TEUCHOS_ASSERT(LIDs[i]>=0);
}
// loop over the fields to be gathered
Teuchos::ArrayRCP<const double> local_x;
for (std::size_t fieldIndex=0; fieldIndex<gatherFields_.size();fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int indexerId = gidIndexer_->getFieldBlock(fieldNum);
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_x = rcp_dynamic_cast<SpmdVectorBase<double> >(x->getNonconstVectorBlock(indexerId));
block_x->getLocalData(ptrFromRef(local_x));
const std::vector<int> & elmtOffset = gidIndexer_->getGIDFieldOffsets(blockId,fieldNum);
// loop over basis functions and fill the fields
for(std::size_t basis=0;basis<elmtOffset.size();basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
// TEUCHOS_ASSERT(indexerId==GIDs[offset].first);
// TEUCHOS_ASSERT(lid<local_x.size() && lid>=0);
(gatherFields_[fieldIndex])(worksetCellIndex,basis) = local_x[lid];
}
}
}
}
Teuchos::RCP<const Epetra_Map>
Thyra::get_Epetra_Map(const VectorSpaceBase<double>& vs_in,
const RCP<const Epetra_Comm>& comm)
{
using Teuchos::rcpFromRef;
using Teuchos::rcpFromPtr;
using Teuchos::rcp_dynamic_cast;
using Teuchos::ptrFromRef;
using Teuchos::ptr_dynamic_cast;
const Ptr<const VectorSpaceBase<double> > vs_ptr = ptrFromRef(vs_in);
const Ptr<const SpmdVectorSpaceBase<double> > spmd_vs =
ptr_dynamic_cast<const SpmdVectorSpaceBase<double> >(vs_ptr);
const Ptr<const ProductVectorSpaceBase<double> > &prod_vs =
ptr_dynamic_cast<const ProductVectorSpaceBase<double> >(vs_ptr);
TEUCHOS_TEST_FOR_EXCEPTION( is_null(spmd_vs) && is_null(prod_vs), std::logic_error,
"Error, the concrete VectorSpaceBase object of type "
+Teuchos::demangleName(typeid(vs_in).name())+" does not support the"
" SpmdVectorSpaceBase or the ProductVectorSpaceBase interfaces!" );
const int numBlocks = (nonnull(prod_vs) ? prod_vs->numBlocks() : 1);
// Get an array of SpmdVectorBase objects for the blocks
Array<RCP<const SpmdVectorSpaceBase<double> > > spmd_vs_blocks;
if (nonnull(prod_vs)) {
for (int block_i = 0; block_i < numBlocks; ++block_i) {
const RCP<const SpmdVectorSpaceBase<double> > spmd_vs_i =
rcp_dynamic_cast<const SpmdVectorSpaceBase<double> >(
prod_vs->getBlock(block_i), true);
spmd_vs_blocks.push_back(spmd_vs_i);
}
}
else {
spmd_vs_blocks.push_back(rcpFromPtr(spmd_vs));
}
// Find the number of local elements, summed over all blocks
int myLocalElements = 0;
for (int block_i = 0; block_i < numBlocks; ++block_i) {
myLocalElements += spmd_vs_blocks[block_i]->localSubDim();
}
// Find the GIDs owned by this processor, taken from all blocks
int count=0;
int blockOffset = 0;
Array<int> myGIDs(myLocalElements);
for (int block_i = 0; block_i < numBlocks; ++block_i) {
const RCP<const SpmdVectorSpaceBase<double> > spmd_vs_i = spmd_vs_blocks[block_i];
const int lowGIDInBlock = spmd_vs_i->localOffset();
const int numLocalElementsInBlock = spmd_vs_i->localSubDim();
for (int i=0; i < numLocalElementsInBlock; ++i, ++count) {
myGIDs[count] = blockOffset + lowGIDInBlock + i;
}
blockOffset += spmd_vs_i->dim();
}
const int globalDim = vs_in.dim();
return Teuchos::rcp(
new Epetra_Map(globalDim, myLocalElements, &(myGIDs[0]), 0, *comm));
}
void panzer::ScatterDirichletResidual_BlockedEpetra<panzer::Traits::Jacobian, Traits,LO,GO>::
evaluateFields(typename Traits::EvalData workset)
{
using Teuchos::RCP;
using Teuchos::ArrayRCP;
using Teuchos::ptrFromRef;
using Teuchos::rcp_dynamic_cast;
using Thyra::VectorBase;
using Thyra::SpmdVectorBase;
using Thyra::ProductVectorBase;
using Thyra::BlockedLinearOpBase;
typedef BlockedEpetraLinearObjContainer BLOC;
std::vector<std::pair<int,GO> > GIDs;
std::vector<int> LIDs;
// for convenience pull out some objects from workset
std::string blockId = workset.block_id;
const std::vector<std::size_t> & localCellIds = workset.cell_local_ids;
RCP<const BLOC> blockedContainer = blockedContainer_;
RCP<ProductVectorBase<double> > r = rcp_dynamic_cast<ProductVectorBase<double> >(blockedContainer->get_f());
Teuchos::RCP<BlockedLinearOpBase<double> > Jac = rcp_dynamic_cast<BlockedLinearOpBase<double> >(blockedContainer->get_A());
int numFieldBlocks = globalIndexer_->getNumFieldBlocks();
std::vector<int> blockOffsets(numFieldBlocks+1); // number of fields, plus a sentinnel
for(int blk=0;blk<numFieldBlocks;blk++) {
int blockOffset = globalIndexer_->getBlockGIDOffset(blockId,blk);
blockOffsets[blk] = blockOffset;
}
boost::unordered_map<std::pair<int,int>,Teuchos::RCP<Epetra_CrsMatrix> > jacEpetraBlocks;
// NOTE: A reordering of these loops will likely improve performance
// The "getGIDFieldOffsets may be expensive. However the
// "getElementGIDs" can be cheaper. However the lookup for LIDs
// may be more expensive!
// scatter operation for each cell in workset
for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
std::size_t cellLocalId = localCellIds[worksetCellIndex];
globalIndexer_->getElementGIDs(cellLocalId,GIDs);
blockOffsets[numFieldBlocks] = GIDs.size();
// caculate the local IDs for this element
LIDs.resize(GIDs.size());
for(std::size_t i=0;i<GIDs.size();i++) {
// used for doing local ID lookups
RCP<const Epetra_Map> r_map = blockedContainer->getMapForBlock(GIDs[i].first);
LIDs[i] = r_map->LID(GIDs[i].second);
}
std::vector<bool> is_owned(GIDs.size(), false);
globalIndexer_->ownedIndices(GIDs,is_owned);
// loop over each field to be scattered
Teuchos::ArrayRCP<double> local_r, local_dc;
for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) {
int fieldNum = fieldIds_[fieldIndex];
int blockRowIndex = globalIndexer_->getFieldBlock(fieldNum);
RCP<SpmdVectorBase<double> > dc = rcp_dynamic_cast<SpmdVectorBase<double> >(dirichletCounter_->getNonconstVectorBlock(blockRowIndex));
dc->getNonconstLocalData(ptrFromRef(local_dc));
// grab local data for inputing
RCP<SpmdVectorBase<double> > block_r = rcp_dynamic_cast<SpmdVectorBase<double> >(r->getNonconstVectorBlock(blockRowIndex));
block_r->getNonconstLocalData(ptrFromRef(local_r));
// this call "should" get the right ordering accordint to the Intrepid basis
const std::pair<std::vector<int>,std::vector<int> > & indicePair
= globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_);
const std::vector<int> & elmtOffset = indicePair.first;
const std::vector<int> & basisIdMap = indicePair.second;
// loop over basis functions
for(std::size_t basis=0;basis<elmtOffset.size();basis++) {
int offset = elmtOffset[basis];
int lid = LIDs[offset];
if(lid<0) // not on this processor
continue;
int basisId = basisIdMap[basis];
if (checkApplyBC_)
if (!applyBC_[fieldIndex](worksetCellIndex,basisId))
continue;
// zero out matrix row
for(int blockColIndex=0;blockColIndex<numFieldBlocks;blockColIndex++) {
int start = blockOffsets[blockColIndex];
int end = blockOffsets[blockColIndex+1];
if(end-start<=0)
continue;
// check hash table for jacobian sub block
std::pair<int,int> blockIndex = std::make_pair(blockRowIndex,blockColIndex);
Teuchos::RCP<Epetra_CrsMatrix> subJac = jacEpetraBlocks[blockIndex];
// if you didn't find one before, add it to the hash table
if(subJac==Teuchos::null) {
Teuchos::RCP<Thyra::LinearOpBase<double> > tOp = Jac->getNonconstBlock(blockIndex.first,blockIndex.second);
// block operator is null, don't do anything (it is excluded)
if(Teuchos::is_null(tOp))
continue;
Teuchos::RCP<Epetra_Operator> eOp = Thyra::get_Epetra_Operator(*tOp);
subJac = rcp_dynamic_cast<Epetra_CrsMatrix>(eOp,true);
jacEpetraBlocks[blockIndex] = subJac;
}
int numEntries = 0;
int * rowIndices = 0;
double * rowValues = 0;
subJac->ExtractMyRowView(lid,numEntries,rowValues,rowIndices);
for(int i=0;i<numEntries;i++)
rowValues[i] = 0.0;
}
const ScalarT & scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,basisId);
local_r[lid] = scatterField.val();
local_dc[lid] = 1.0; // mark row as dirichlet
// loop over the sensitivity indices: all DOFs on a cell
std::vector<double> jacRow(scatterField.size(),0.0);
for(int sensIndex=0;sensIndex<scatterField.size();++sensIndex)
jacRow[sensIndex] = scatterField.fastAccessDx(sensIndex);
TEUCHOS_ASSERT(jacRow.size()==GIDs.size());
for(int blockColIndex=0;blockColIndex<numFieldBlocks;blockColIndex++) {
int start = blockOffsets[blockColIndex];
int end = blockOffsets[blockColIndex+1];
if(end-start<=0)
continue;
// check hash table for jacobian sub block
std::pair<int,int> blockIndex = std::make_pair(blockRowIndex,blockColIndex);
Teuchos::RCP<Epetra_CrsMatrix> subJac = jacEpetraBlocks[blockIndex];
// if you didn't find one before, add it to the hash table
if(subJac==Teuchos::null) {
Teuchos::RCP<Thyra::LinearOpBase<double> > tOp = Jac->getNonconstBlock(blockIndex.first,blockIndex.second);
// block operator is null, don't do anything (it is excluded)
if(Teuchos::is_null(tOp))
continue;
Teuchos::RCP<Epetra_Operator> eOp = Thyra::get_Epetra_Operator(*tOp);
subJac = rcp_dynamic_cast<Epetra_CrsMatrix>(eOp,true);
jacEpetraBlocks[blockIndex] = subJac;
}
// Sum Jacobian
int err = subJac->ReplaceMyValues(lid, end-start, &jacRow[start],&LIDs[start]);
if(err!=0) {
std::stringstream ss;
ss << "Failed inserting row: " << GIDs[offset].second << " (" << lid << "): ";
for(int i=start;i<end;i++)
ss << GIDs[i].second << " (" << LIDs[i] << ") ";
ss << std::endl;
ss << "Into block " << blockRowIndex << ", " << blockColIndex << std::endl;
ss << "scatter field = ";
scatterFields_[fieldIndex].print(ss);
ss << std::endl;
TEUCHOS_TEST_FOR_EXCEPTION(err!=0,std::runtime_error,ss.str());
}
}
}
}
}
}