void create_nb_indices_per_row(cf3::common::PE::CommPattern& cp,
const VariablesDescriptor& variables,
const std::vector<Uint>& starting_indices,
std::vector<int>& num_indices_per_row
)
{
const Uint nb_vars = variables.nb_vars();
const Uint total_nb_eq = variables.size();
const Uint nb_nodes_for_rank = cp.isUpdatable().size();
cf3_assert(nb_nodes_for_rank+1 == starting_indices.size());
num_indices_per_row.reserve(nb_nodes_for_rank*total_nb_eq);
for(Uint var_idx = 0; var_idx != nb_vars; ++var_idx)
{
const Uint neq = variables.var_length(var_idx);
const Uint var_offset = variables.offset(var_idx);
for (int i=0; i<nb_nodes_for_rank; i++)
{
if (cp.isUpdatable()[i])
{
for(int j = 0; j != neq; ++j)
{
num_indices_per_row.push_back(total_nb_eq*(starting_indices[i+1]-starting_indices[i]));
}
}
}
}
}
void TrilinosMatrix::create(cf3::common::PE::CommPattern& cp, Uint neq, std::vector<Uint>& node_connectivity, std::vector<Uint>& starting_indices, LSS::Vector& solution, LSS::Vector& rhs)
{
/// @todo structurally symmetricize the matrix
/// @todo ensure main diagonal blocks always existent
// if already created
if (m_is_created) destroy();
// get global ids vector
int *gid=(int*)cp.gid()->pack();
// prepare intermediate data
int nmyglobalelements=0;
int maxrowentries=0;
std::vector<int> rowelements(0);
std::vector<int> myglobalelements(0);
for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
if (cp.isUpdatable()[i])
{
++nmyglobalelements;
myglobalelements.push_back((int)gid[i]);
rowelements.push_back((int)(starting_indices[i+1]-starting_indices[i]));
maxrowentries=maxrowentries<(starting_indices[i+1]-starting_indices[i])?(starting_indices[i+1]-starting_indices[i]):maxrowentries;
}
std::vector<double>dummy_entries(maxrowentries*neq*neq,0.);
std::vector<int>global_columns(maxrowentries);
// process local to matrix local numbering mapper
int iupd=0;
int ighost=nmyglobalelements;
m_p2m.resize(0);
m_p2m.reserve(cp.isUpdatable().size());
for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
{
if (cp.isUpdatable()[i]) { m_p2m.push_back(iupd++); }
else { m_p2m.push_back(ighost++); }
}
// blockmaps (colmap is gid 1 to 1, rowmap is gid with ghosts filtered out)
Epetra_BlockMap rowmap(-1,nmyglobalelements,&myglobalelements[0],neq,0,m_comm);
for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
if (!cp.isUpdatable()[i])
myglobalelements.push_back((int)gid[i]);
Epetra_BlockMap colmap(-1,cp.isUpdatable().size(),&myglobalelements[0],neq,0,m_comm);
myglobalelements.clear();
// create matrix
m_mat=Teuchos::rcp(new Epetra_FEVbrMatrix(Copy,rowmap,colmap,&rowelements[0]));
/*must be a bug in Trilinos, Epetra_FEVbrMatrix constructor is in Copy mode but it hangs up anyway
more funny, when it gets out of scope and gets dealloc'd, everything survives according to memcheck
rowmap.~Epetra_BlockMap();
colmap.~Epetra_BlockMap();
*/
rowelements.clear();
// prepare the entries
for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
if (cp.isUpdatable()[i])
{
// for(int j=(const int)starting_indices[i]; j<(const int)starting_indices[i+1]; j++) global_columns[j-starting_indices[i]]=gid[m_p2m[node_connectivity[j]]];
for(int j=(const int)starting_indices[i]; j<(const int)starting_indices[i+1]; j++) global_columns[j-starting_indices[i]]=myglobalelements[m_p2m[node_connectivity[j]]];
TRILINOS_THROW(m_mat->BeginInsertGlobalValues(gid[i],(int)(starting_indices[i+1]-starting_indices[i]),&global_columns[0]));
for(int j=(const int)starting_indices[i]; j<(const int)starting_indices[i+1]; j++)
TRILINOS_THROW(m_mat->SubmitBlockEntry(&dummy_entries[0],0,neq,neq));
TRILINOS_THROW(m_mat->EndSubmitEntries());
}
TRILINOS_THROW(m_mat->FillComplete());
TRILINOS_THROW(m_mat->OptimizeStorage()); // in theory fillcomplete calls optimizestorage from Trilinos 8.x+
delete[] gid;
// set class properties
m_is_created=true;
m_neq=neq;
m_blockrow_size=nmyglobalelements;
m_blockcol_size=cp.gid()->size();
}
void TrilinosFEVbrMatrix::create(cf3::common::PE::CommPattern& cp, const Uint neq, const std::vector<Uint>& node_connectivity, const std::vector<Uint>& starting_indices, LSS::Vector& solution, LSS::Vector& rhs, const std::vector<Uint>& periodic_links_nodes, const std::vector<bool>& periodic_links_active)
{
/// @todo structurally symmetricize the matrix
/// @todo ensure main diagonal blocks always existent
// if already created
if (m_is_created) destroy();
// Copy node connectivity
m_node_connectivity.resize(node_connectivity.size());
m_starting_indices.resize(starting_indices.size());
std::copy(node_connectivity.begin(), node_connectivity.end(), m_node_connectivity.begin());
std::copy(starting_indices.begin(), starting_indices.end(), m_starting_indices.begin());
const int nb_nodes = cp.isUpdatable().size();
std::vector<int> myglobalelements(0);
std::vector<Uint> my_ranks;
int nmyglobalelements=0;
boost::shared_ptr<VariablesDescriptor> single_var_descriptor = common::allocate_component<VariablesDescriptor>("SingleVariableDescriptor");
single_var_descriptor->options().set(common::Tags::dimension(), 1); // Use one equation that represents the entire block
single_var_descriptor->push_back("LSSvars", VariablesDescriptor::Dimensionalities::VECTOR);
create_map_data(cp, *single_var_descriptor, m_p2m, myglobalelements, my_ranks, nmyglobalelements, periodic_links_nodes, periodic_links_active);
std::vector<int> rowelements; rowelements.reserve(nmyglobalelements);
std::vector<int> indices_per_row;
create_indices_per_row(cp, *single_var_descriptor, node_connectivity, starting_indices, m_p2m, rowelements, indices_per_row, periodic_links_nodes, periodic_links_active);
const int maxrowentries = *std::max_element(rowelements.begin(), rowelements.end());
std::vector<double>dummy_entries(maxrowentries*neq*neq,0.);
// blockmaps (colmap is gid 1 to 1, rowmap is gid with ghosts filtered out)
Epetra_BlockMap rowmap(-1,nmyglobalelements,&myglobalelements[0],neq,0,m_comm);
Epetra_BlockMap colmap(-1,myglobalelements.size(),&myglobalelements[0],neq,0,m_comm);
myglobalelements.clear();
// create matrix
m_mat=Teuchos::rcp(new Epetra_FEVbrMatrix(Copy,rowmap,colmap,&rowelements[0]));
// prepare the entries
int row_start = 0;
cf3_assert(rowelements.size() == nmyglobalelements);
for(int i = 0; i != nmyglobalelements; ++i)
{
const int row_nb_elems = rowelements[i];
cf3_assert( (row_start + row_nb_elems) <= indices_per_row.size() );
TRILINOS_THROW(m_mat->BeginInsertMyValues(i,row_nb_elems,&indices_per_row[row_start]));
for(int j=0; j<row_nb_elems; j++)
{
TRILINOS_THROW(m_mat->SubmitBlockEntry(&dummy_entries[0],0,neq,neq));
}
TRILINOS_THROW(m_mat->EndSubmitEntries());
row_start += row_nb_elems;
}
TRILINOS_THROW(m_mat->FillComplete());
// set class properties
m_is_created=true;
m_neq=neq;
m_blockrow_size=nmyglobalelements;
m_blockcol_size=cp.gid()->size();
CFdebug << "Created a " << m_mat->NumGlobalCols() << " x " << m_mat->NumGlobalRows() << " trilinos matrix with " << m_mat->NumGlobalNonzeros() << " non-zero elements. CrsGraph rows: " << m_mat->Graph().NumGlobalRows() << CFendl;
}
