void create_nb_indices_per_row(cf3::common::PE::CommPattern& cp,
                     const VariablesDescriptor& variables,
                     const std::vector<Uint>& starting_indices,
                     std::vector<int>& num_indices_per_row
                    )
{
  const Uint nb_vars = variables.nb_vars();
  const Uint total_nb_eq = variables.size();

  const Uint nb_nodes_for_rank = cp.isUpdatable().size();
  cf3_assert(nb_nodes_for_rank+1 == starting_indices.size());
  num_indices_per_row.reserve(nb_nodes_for_rank*total_nb_eq);

  for(Uint var_idx = 0; var_idx != nb_vars; ++var_idx)
  {
    const Uint neq = variables.var_length(var_idx);
    const Uint var_offset = variables.offset(var_idx);
    for (int i=0; i<nb_nodes_for_rank; i++)
    {
      if (cp.isUpdatable()[i])
      {
        for(int j = 0; j != neq; ++j)
        {
          num_indices_per_row.push_back(total_nb_eq*(starting_indices[i+1]-starting_indices[i]));
        }
      }
    }
  }
}
Example #2
0
void TrilinosMatrix::create(cf3::common::PE::CommPattern& cp, Uint neq, std::vector<Uint>& node_connectivity, std::vector<Uint>& starting_indices, LSS::Vector& solution, LSS::Vector& rhs)
{
  /// @todo structurally symmetricize the matrix
  /// @todo ensure main diagonal blocks always existent

  // if already created
  if (m_is_created) destroy();

  // get global ids vector
  int *gid=(int*)cp.gid()->pack();

  // prepare intermediate data
  int nmyglobalelements=0;
  int maxrowentries=0;
  std::vector<int> rowelements(0);
  std::vector<int> myglobalelements(0);

  for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
    if (cp.isUpdatable()[i])
    {
      ++nmyglobalelements;
      myglobalelements.push_back((int)gid[i]);
      rowelements.push_back((int)(starting_indices[i+1]-starting_indices[i]));
      maxrowentries=maxrowentries<(starting_indices[i+1]-starting_indices[i])?(starting_indices[i+1]-starting_indices[i]):maxrowentries;
    }
  std::vector<double>dummy_entries(maxrowentries*neq*neq,0.);
  std::vector<int>global_columns(maxrowentries);

  // process local to matrix local numbering mapper
  int iupd=0;
  int ighost=nmyglobalelements;
  m_p2m.resize(0);
  m_p2m.reserve(cp.isUpdatable().size());
  for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
  {
    if (cp.isUpdatable()[i]) { m_p2m.push_back(iupd++); }
    else { m_p2m.push_back(ighost++); }
  }

  // blockmaps (colmap is gid 1 to 1, rowmap is gid with ghosts filtered out)
  Epetra_BlockMap rowmap(-1,nmyglobalelements,&myglobalelements[0],neq,0,m_comm);
  for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
    if (!cp.isUpdatable()[i])
      myglobalelements.push_back((int)gid[i]);
  Epetra_BlockMap colmap(-1,cp.isUpdatable().size(),&myglobalelements[0],neq,0,m_comm);
  myglobalelements.clear();

  // create matrix
  m_mat=Teuchos::rcp(new Epetra_FEVbrMatrix(Copy,rowmap,colmap,&rowelements[0]));
/*must be a bug in Trilinos, Epetra_FEVbrMatrix constructor is in Copy mode but it hangs up anyway
  more funny, when it gets out of scope and gets dealloc'd, everything survives according to memcheck
  rowmap.~Epetra_BlockMap();
  colmap.~Epetra_BlockMap();
*/
  rowelements.clear();

  // prepare the entries
  for (int i=0; i<(const int)cp.isUpdatable().size(); i++)
    if (cp.isUpdatable()[i])
    {
//      for(int j=(const int)starting_indices[i]; j<(const int)starting_indices[i+1]; j++) global_columns[j-starting_indices[i]]=gid[m_p2m[node_connectivity[j]]];
      for(int j=(const int)starting_indices[i]; j<(const int)starting_indices[i+1]; j++) global_columns[j-starting_indices[i]]=myglobalelements[m_p2m[node_connectivity[j]]];
      TRILINOS_THROW(m_mat->BeginInsertGlobalValues(gid[i],(int)(starting_indices[i+1]-starting_indices[i]),&global_columns[0]));
      for(int j=(const int)starting_indices[i]; j<(const int)starting_indices[i+1]; j++)
        TRILINOS_THROW(m_mat->SubmitBlockEntry(&dummy_entries[0],0,neq,neq));
      TRILINOS_THROW(m_mat->EndSubmitEntries());
    }
  TRILINOS_THROW(m_mat->FillComplete());
  TRILINOS_THROW(m_mat->OptimizeStorage()); // in theory fillcomplete calls optimizestorage from Trilinos 8.x+
  delete[] gid;

  // set class properties
  m_is_created=true;
  m_neq=neq;
  m_blockrow_size=nmyglobalelements;
  m_blockcol_size=cp.gid()->size();
}
void TrilinosFEVbrMatrix::create(cf3::common::PE::CommPattern& cp, const Uint neq, const std::vector<Uint>& node_connectivity, const std::vector<Uint>& starting_indices, LSS::Vector& solution, LSS::Vector& rhs, const std::vector<Uint>& periodic_links_nodes, const std::vector<bool>& periodic_links_active)
{
  /// @todo structurally symmetricize the matrix
  /// @todo ensure main diagonal blocks always existent

  // if already created
  if (m_is_created) destroy();

  // Copy node connectivity
  m_node_connectivity.resize(node_connectivity.size());
  m_starting_indices.resize(starting_indices.size());
  std::copy(node_connectivity.begin(), node_connectivity.end(), m_node_connectivity.begin());
  std::copy(starting_indices.begin(), starting_indices.end(), m_starting_indices.begin());

  const int nb_nodes = cp.isUpdatable().size();

  std::vector<int> myglobalelements(0);
  std::vector<Uint> my_ranks;
  int nmyglobalelements=0;
  boost::shared_ptr<VariablesDescriptor> single_var_descriptor = common::allocate_component<VariablesDescriptor>("SingleVariableDescriptor");
  single_var_descriptor->options().set(common::Tags::dimension(), 1); // Use one equation that represents the entire block
  single_var_descriptor->push_back("LSSvars", VariablesDescriptor::Dimensionalities::VECTOR);
  create_map_data(cp, *single_var_descriptor, m_p2m, myglobalelements, my_ranks, nmyglobalelements, periodic_links_nodes, periodic_links_active);

  std::vector<int> rowelements; rowelements.reserve(nmyglobalelements);
  std::vector<int> indices_per_row;
  create_indices_per_row(cp, *single_var_descriptor, node_connectivity, starting_indices, m_p2m, rowelements, indices_per_row, periodic_links_nodes, periodic_links_active);
  const int maxrowentries = *std::max_element(rowelements.begin(), rowelements.end());

  std::vector<double>dummy_entries(maxrowentries*neq*neq,0.);

  // blockmaps (colmap is gid 1 to 1, rowmap is gid with ghosts filtered out)
  Epetra_BlockMap rowmap(-1,nmyglobalelements,&myglobalelements[0],neq,0,m_comm);
  Epetra_BlockMap colmap(-1,myglobalelements.size(),&myglobalelements[0],neq,0,m_comm);
  myglobalelements.clear();

  // create matrix
  m_mat=Teuchos::rcp(new Epetra_FEVbrMatrix(Copy,rowmap,colmap,&rowelements[0]));
  
  // prepare the entries
  int row_start = 0;
  cf3_assert(rowelements.size() == nmyglobalelements);
  for(int i = 0; i != nmyglobalelements; ++i)
  {
    const int row_nb_elems = rowelements[i];
    cf3_assert( (row_start + row_nb_elems) <= indices_per_row.size() );
    TRILINOS_THROW(m_mat->BeginInsertMyValues(i,row_nb_elems,&indices_per_row[row_start]));
    for(int j=0; j<row_nb_elems; j++)
    {
      TRILINOS_THROW(m_mat->SubmitBlockEntry(&dummy_entries[0],0,neq,neq));
    }
    TRILINOS_THROW(m_mat->EndSubmitEntries());
    row_start += row_nb_elems;
  }
  
  TRILINOS_THROW(m_mat->FillComplete());

  // set class properties
  m_is_created=true;
  m_neq=neq;
  m_blockrow_size=nmyglobalelements;
  m_blockcol_size=cp.gid()->size();
  CFdebug << "Created a " << m_mat->NumGlobalCols() << " x " << m_mat->NumGlobalRows() << " trilinos matrix with " << m_mat->NumGlobalNonzeros() << " non-zero elements. CrsGraph rows: " << m_mat->Graph().NumGlobalRows() << CFendl;
}