Albany::OrdinarySTKFieldContainer<Interleaved>::OrdinarySTKFieldContainer(
  const Teuchos::RCP<Teuchos::ParameterList>& params_,
  const Teuchos::RCP<stk::mesh::MetaData>& metaData_,
  const Teuchos::RCP<stk::mesh::BulkData>& bulkData_,
  const int neq_,
  const AbstractFieldContainer::FieldContainerRequirements& req,
  const int numDim_,
  const Teuchos::RCP<Albany::StateInfoStruct>& sis)
  : GenericSTKFieldContainer<Interleaved>(params_, metaData_, bulkData_, neq_, numDim_),
      buildSphereVolume(false) {

  typedef typename AbstractSTKFieldContainer::VectorFieldType VFT;
  typedef typename AbstractSTKFieldContainer::ScalarFieldType SFT;
  typedef typename AbstractSTKFieldContainer::SphereVolumeFieldType SVFT;

  int num_time_deriv = params_->get<int>("Number Of Time Derivatives");

  //Start STK stuff
  this->coordinates_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, "coordinates");
  stk::mesh::put_field(*this->coordinates_field , metaData_->universal_part(), numDim_);
#ifdef ALBANY_SEACAS
  stk::io::set_field_role(*this->coordinates_field, Ioss::Field::MESH);
#endif

  solution_field.resize(num_time_deriv + 1);
  solution_field_dtk.resize(num_time_deriv + 1);

  for(int num_vecs = 0; num_vecs <= num_time_deriv; num_vecs++){

    solution_field[num_vecs] = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                    params_->get<std::string>(sol_tag_name[num_vecs], sol_id_name[num_vecs]));

    stk::mesh::put_field(*solution_field[num_vecs] , metaData_->universal_part(), neq_);

#if defined(ALBANY_DTK)
    solution_field_dtk[num_vecs] = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                    params_->get<std::string>(res_tag_name[num_vecs], sol_dtk_id_name[num_vecs]));
    stk::mesh::put_field(*solution_field_dtk[num_vecs] , metaData_->universal_part() , neq_);
#endif

#ifdef ALBANY_SEACAS
    stk::io::set_field_role(*solution_field[num_vecs], Ioss::Field::TRANSIENT);
#if defined(ALBANY_DTK)
    stk::io::set_field_role(*solution_field_dtk[num_vecs], Ioss::Field::TRANSIENT);
#endif
#endif

  }

#if defined(ALBANY_LCM)
  residual_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                    params_->get<std::string>(res_tag_name[0], res_id_name[0]));
  stk::mesh::put_field(*residual_field, metaData_->universal_part() , neq_);
#ifdef ALBANY_SEACAS
  stk::io::set_field_role(*residual_field, Ioss::Field::TRANSIENT);
#endif
#endif

#if defined(ALBANY_LCM) && defined(ALBANY_SEACAS)
  // sphere volume is a mesh attribute read from a genesis mesh file containing sphere element (used for peridynamics)
  bool hasSphereVolumeFieldContainerRequirement = (std::find(req.begin(), req.end(), "Sphere Volume") != req.end());
  if(hasSphereVolumeFieldContainerRequirement){
    this->sphereVolume_field = metaData_->template get_field< SVFT >(stk::topology::ELEMENT_RANK, "volume");
    if(this->sphereVolume_field != 0){
      buildSphereVolume = true;
      stk::io::set_field_role(*this->sphereVolume_field, Ioss::Field::ATTRIBUTE);
    }
  }
#endif

  // If the problem requests that the initial guess at the solution equals the input node coordinates,
  // set that here
  /*
    if(std::find(req.begin(), req.end(), "Initial Guess Coords") != req.end()){
       this->copySTKField(this->coordinates_field, solution_field);
    }
  */

  this->addStateStructs(sis);

  initializeSTKAdaptation();
}
void Albany::ExtrudedSTKMeshStruct::setFieldAndBulkData(
    const Teuchos::RCP<const Teuchos_Comm>& comm,
    const Teuchos::RCP<Teuchos::ParameterList>& params,
    const unsigned int neq_,
    const AbstractFieldContainer::FieldContainerRequirements& req,
    const Teuchos::RCP<Albany::StateInfoStruct>& sis,
    const unsigned int worksetSize)
{
  int numLayers = params->get("NumLayers", 10);
  bool useGlimmerSpacing = params->get("Use Glimmer Spacing", false);
  GO maxGlobalElements2D = 0;
  GO maxGlobalVertices2dId = 0;
  GO numGlobalVertices2D = 0;
  GO maxGlobalEdges2D = 0;
  bool Ordering = params->get("Columnwise Ordering", false);
  bool isTetra = true;

  stk::mesh::BulkData& bulkData2D = *basalMeshStruct->bulkData;
  stk::mesh::MetaData& metaData2D = *basalMeshStruct->metaData; //bulkData2D.mesh_meta_data();

  std::vector<double> levelsNormalizedThickness(numLayers + 1), temperatureNormalizedZ, flowRateNormalizedZ;

  if(useGlimmerSpacing)
    for (int i = 0; i < numLayers+1; i++)
      levelsNormalizedThickness[numLayers-i] = 1.0- (1.0 - std::pow(double(i) / numLayers + 1.0, -2))/(1.0 - std::pow(2.0, -2));
  else  //uniform layers
    for (int i = 0; i < numLayers+1; i++)
      levelsNormalizedThickness[i] = double(i) / numLayers;

  Teuchos::ArrayRCP<double> layerThicknessRatio(numLayers), layersNormalizedThickness(numLayers);
  for (int i = 0; i < numLayers; i++) {
    layerThicknessRatio[i] = levelsNormalizedThickness[i+1]-levelsNormalizedThickness[i];
    layersNormalizedThickness[i] = 0.5*(levelsNormalizedThickness[i]+levelsNormalizedThickness[i+1]);
  }

  /*std::cout<< "Levels: ";
  for (int i = 0; i < numLayers+1; i++)
    std::cout<< levelsNormalizedThickness[i] << " ";
  std::cout<< "\n";*/

  stk::mesh::Selector select_owned_in_part = stk::mesh::Selector(metaData2D.universal_part()) & stk::mesh::Selector(metaData2D.locally_owned_part());

  stk::mesh::Selector select_overlap_in_part = stk::mesh::Selector(metaData2D.universal_part()) & (stk::mesh::Selector(metaData2D.locally_owned_part()) | stk::mesh::Selector(metaData2D.globally_shared_part()));

  stk::mesh::Selector select_edges = stk::mesh::Selector(*metaData2D.get_part("LateralSide")) & (stk::mesh::Selector(metaData2D.locally_owned_part()) | stk::mesh::Selector(metaData2D.globally_shared_part()));

  std::vector<stk::mesh::Entity> cells2D;
  stk::mesh::get_selected_entities(select_overlap_in_part, bulkData2D.buckets(stk::topology::ELEMENT_RANK), cells2D);

  std::vector<stk::mesh::Entity> nodes2D;
  stk::mesh::get_selected_entities(select_overlap_in_part, bulkData2D.buckets(stk::topology::NODE_RANK), nodes2D);

  std::vector<stk::mesh::Entity> edges2D;
  stk::mesh::get_selected_entities(select_edges, bulkData2D.buckets(metaData2D.side_rank()), edges2D);

  GO maxOwnedElements2D(0), maxOwnedNodes2D(0), maxOwnedSides2D(0), numOwnedNodes2D(0);
  for (int i = 0; i < cells2D.size(); i++)
    maxOwnedElements2D = std::max(maxOwnedElements2D, (GO) bulkData2D.identifier(cells2D[i]));
  for (int i = 0; i < nodes2D.size(); i++)
    maxOwnedNodes2D = std::max(maxOwnedNodes2D, (GO) bulkData2D.identifier(nodes2D[i]));
  for (int i = 0; i < edges2D.size(); i++)
    maxOwnedSides2D = std::max(maxOwnedSides2D, (GO) bulkData2D.identifier(edges2D[i]));
  numOwnedNodes2D = stk::mesh::count_selected_entities(select_owned_in_part, bulkData2D.buckets(stk::topology::NODE_RANK));


  //WARNING Currently GO == long int. For gcc compiler, long == long long, however this might not be true with other compilers.

  //comm->MaxAll(&maxOwnedElements2D, &maxGlobalElements2D, 1);
  Teuchos::reduceAll<int, GO>(*comm, Teuchos::REDUCE_MAX, maxOwnedElements2D, Teuchos::ptr(&maxGlobalElements2D));
  //comm->MaxAll(&maxOwnedNodes2D, &maxGlobalVertices2dId, 1);
  Teuchos::reduceAll<int, GO>(*comm, Teuchos::REDUCE_MAX, maxOwnedNodes2D, Teuchos::ptr(&maxGlobalVertices2dId));
  //comm->MaxAll(&maxOwnedSides2D, &maxGlobalEdges2D, 1);
  Teuchos::reduceAll<int, GO>(*comm, Teuchos::REDUCE_MAX, maxOwnedSides2D, Teuchos::ptr(&maxGlobalEdges2D));
  //comm->SumAll(&numOwnedNodes2D, &numGlobalVertices2D, 1);
  //The following should not be int int...
  Teuchos::reduceAll<int, GO>(*comm, Teuchos::REDUCE_SUM, 1, &numOwnedNodes2D, &numGlobalVertices2D);

  if (comm->getRank() == 0) std::cout << "Importing ascii files ...";

  //std::cout << "Num Global Elements: " << maxGlobalElements2D<< " " << maxGlobalVertices2dId<< " " << maxGlobalEdges2D << std::endl;

  Teuchos::Array<GO> indices(nodes2D.size());
  for (int i = 0; i < nodes2D.size(); ++i)
    indices[i] = bulkData2D.identifier(nodes2D[i]) - 1;

  Teuchos::RCP<const Tpetra_Map>
    nodes_map = Tpetra::createNonContigMapWithNode<LO, GO> (
      indices(), comm, KokkosClassic::Details::getNode<KokkosNode>());
  int numMyElements = (comm->getRank() == 0) ? numGlobalVertices2D : 0;
  //Teuchos::RCP<const Tpetra_Map> serial_nodes_map = Tpetra::createUniformContigMap<LO, GO>(numMyElements, comm);
  Teuchos::RCP<const Tpetra_Map> serial_nodes_map = Teuchos::rcp(new const Tpetra_Map(INVALID, numMyElements, 0, comm));
  Teuchos::RCP<Tpetra_Import> importOperator = Teuchos::rcp(new Tpetra_Import(serial_nodes_map, nodes_map));

  Teuchos::RCP<Tpetra_Vector> temp = Teuchos::rcp(new Tpetra_Vector(serial_nodes_map));
  Teuchos::RCP<Tpetra_Vector> sHeightVec;
  Teuchos::RCP<Tpetra_Vector> thickVec;
  Teuchos::RCP<Tpetra_Vector> bTopographyVec;
  Teuchos::RCP<Tpetra_Vector> bFrictionVec;
  Teuchos::RCP<Tpetra_MultiVector> temperatureVecInterp;
  Teuchos::RCP<Tpetra_MultiVector> flowRateVecInterp;
  Teuchos::RCP<Tpetra_MultiVector> sVelocityVec;
  Teuchos::RCP<Tpetra_MultiVector> velocityRMSVec;



  bool hasSurface_height =  std::find(req.begin(), req.end(), "surface_height") != req.end();

  {
    sHeightVec = Teuchos::rcp(new Tpetra_Vector(nodes_map));
    std::string fname = params->get<std::string>("Surface Height File Name", "surface_height.ascii");
    read2DFileSerial(fname, temp, comm);
    sHeightVec->doImport(*temp, *importOperator, Tpetra::INSERT);
  }
  Teuchos::ArrayRCP<const ST> sHeightVec_constView = sHeightVec->get1dView();


  bool hasThickness =  std::find(req.begin(), req.end(), "thickness") != req.end();

  {
    std::string fname = params->get<std::string>("Thickness File Name", "thickness.ascii");
    read2DFileSerial(fname, temp, comm);
    thickVec = Teuchos::rcp(new Tpetra_Vector(nodes_map));
    thickVec->doImport(*temp, *importOperator, Tpetra::INSERT);
  }
  Teuchos::ArrayRCP<const ST> thickVec_constView = thickVec->get1dView();

  bool hasBed_topography =  std::find(req.begin(), req.end(), "bed_topography") != req.end();
  {
    std::string fname = params->get<std::string>("Bed Topography File Name", "bed_topography.ascii");
    read2DFileSerial(fname, temp, comm);
    bTopographyVec = Teuchos::rcp(new Tpetra_Vector(nodes_map));
    bTopographyVec->doImport(*temp, *importOperator, Tpetra::INSERT);
  }
  Teuchos::ArrayRCP<const ST> bTopographyVec_constView = bTopographyVec->get1dView();


  bool hasBasal_friction = std::find(req.begin(), req.end(), "basal_friction") != req.end();
  if(hasBasal_friction)
  {
    bFrictionVec = Teuchos::rcp(new Tpetra_Vector(nodes_map));

    if (params->isParameter("Basal Friction File Name"))
    {
      std::string fname = params->get<std::string>("Basal Friction File Name", "basal_friction.ascii");
      read2DFileSerial(fname, temp, comm);
      bFrictionVec->doImport(*temp, *importOperator, Tpetra::INSERT);
    }
    else
    {
      // Try to load it from the 2D mesh
      Albany::AbstractSTKFieldContainer::ScalarFieldType* field = 0;
      field = metaData2D.get_field<Albany::AbstractSTKFieldContainer::ScalarFieldType>(stk::topology::NODE_RANK, "basal_friction");
      if (field!=0)
      {
        Teuchos::ArrayRCP<ST> bFrictionVec_view = bFrictionVec->get1dViewNonConst();

        stk::mesh::Entity node;
        stk::mesh::EntityId nodeId;
        int lid;
        double* values;

        //Now we have to stuff the vector in the mesh data
        for (int i(0); i<nodes2D.size(); ++i)
        {
          nodeId = bulkData2D.identifier(nodes2D[i]) - 1;
          lid    = nodes_map->getLocalElement((GO)(nodeId));

          values = stk::mesh::field_data(*field, nodes2D[i]);
          bFrictionVec_view[lid] = values[0];
        }
      }
      else
      {
        // We use a zero vector, but we issue a warning. Just in case the user forgot to setup something
        std::cout << "No file name specified for 'basal_friction', and no field retrieved from the mesh. Using a zero vector.\n";
      }
    }
  }

  bool hasFlowRate = true;//std::find(req.begin(), req.end(), "flowRate") != req.end();
  if(hasFlowRate) {
    Teuchos::RCP<Tpetra_MultiVector> flowRateVec;
    flowRateVecInterp = Teuchos::rcp(new Tpetra_MultiVector(nodes_map, numLayers));
    std::string fname = params->get<std::string>("Flow Rate File Name", "flow_rate.ascii");
    int err = readFileSerial(fname, serial_nodes_map, nodes_map, importOperator, flowRateVec, flowRateNormalizedZ, comm);

    if(err == 0) {
      int il0, il1, verticalTSize(flowRateVec->getNumVectors());
      double h0(0.0);

      for (int il = 0; il < numLayers; il++) {
        if (layersNormalizedThickness[il] <= flowRateNormalizedZ[0]) {
          il0 = 0;
          il1 = 0;
          h0 = 1.0;
        }

        else if (layersNormalizedThickness[il] >= flowRateNormalizedZ[verticalTSize - 1]) {
          il0 = verticalTSize - 1;
          il1 = verticalTSize - 1;
          h0 = 0.0;
        }

        else {
          int k = 0;
          while (layersNormalizedThickness[il] > flowRateNormalizedZ[++k])
            ;
          il0 = k - 1;
          il1 = k;
          h0 = (flowRateNormalizedZ[il1] - layersNormalizedThickness[il]) / (flowRateNormalizedZ[il1] - flowRateNormalizedZ[il0]);
        }
        Teuchos::ArrayRCP<ST> flowRateVecInterp_nonConstView = flowRateVecInterp->getVectorNonConst(il)->get1dViewNonConst();
        Teuchos::ArrayRCP<const ST> flowRateVec_constView_il0 = flowRateVec->getVectorNonConst(il0)->get1dView();
        Teuchos::ArrayRCP<const ST> flowRateVec_constView_il1 = flowRateVec->getVectorNonConst(il1)->get1dView();

        for (int i = 0; i < nodes_map->getNodeNumElements(); i++)
          flowRateVecInterp_nonConstView[i] = h0 * flowRateVec_constView_il0[i] + (1.0 - h0) * flowRateVec_constView_il1[i];
      }
    }
    else hasFlowRate=false;
  }

  bool hasTemperature = std::find(req.begin(), req.end(), "temperature") != req.end();
  if(hasTemperature) {
    Teuchos::RCP<Tpetra_MultiVector> temperatureVec;
    temperatureVecInterp = Teuchos::rcp(new Tpetra_MultiVector(nodes_map, numLayers + 1));
    std::string fname = params->get<std::string>("Temperature File Name", "temperature.ascii");
    int err = readFileSerial(fname, serial_nodes_map, nodes_map, importOperator, temperatureVec, temperatureNormalizedZ, comm);

    if(err == 0) {
      int il0, il1, verticalTSize(temperatureVec->getNumVectors());
      double h0(0.0);

      for (int il = 0; il < numLayers + 1; il++) {
        if (levelsNormalizedThickness[il] <= temperatureNormalizedZ[0]) {
          il0 = 0;
          il1 = 0;
          h0 = 1.0;
        }

        else if (levelsNormalizedThickness[il] >= temperatureNormalizedZ[verticalTSize - 1]) {
          il0 = verticalTSize - 1;
          il1 = verticalTSize - 1;
          h0 = 0.0;
        }

        else {
          int k = 0;
          while (levelsNormalizedThickness[il] > temperatureNormalizedZ[++k])
            ;
          il0 = k - 1;
          il1 = k;
          h0 = (temperatureNormalizedZ[il1] - levelsNormalizedThickness[il]) / (temperatureNormalizedZ[il1] - temperatureNormalizedZ[il0]);
        }
        Teuchos::ArrayRCP<ST> temperatureVecInterp_nonConstView = temperatureVecInterp->getVectorNonConst(il)->get1dViewNonConst();
        Teuchos::ArrayRCP<const ST> temperatureVec_constView_il0 = temperatureVec->getVectorNonConst(il0)->get1dView();
        Teuchos::ArrayRCP<const ST> temperatureVec_constView_il1 = temperatureVec->getVectorNonConst(il1)->get1dView();

        for (int i = 0; i < nodes_map->getNodeNumElements(); i++)
          temperatureVecInterp_nonConstView[i] = h0 * temperatureVec_constView_il0[i] + (1.0 - h0) * temperatureVec_constView_il1[i];
      }
    }
    else hasTemperature = false;
  }

  Tpetra_MultiVector tempSV(serial_nodes_map,neq_);

  bool hasSurfaceVelocity = std::find(req.begin(), req.end(), "surface_velocity") != req.end();
  if(hasSurfaceVelocity) {
    std::string fname = params->get<std::string>("Surface Velocity File Name", "surface_velocity.ascii");
    readFileSerial(fname, tempSV, comm);
    sVelocityVec = Teuchos::rcp(new Tpetra_MultiVector (nodes_map, neq_));
    sVelocityVec->doImport(tempSV, *importOperator, Tpetra::INSERT);
  }

  bool hasSurfaceVelocityRMS = std::find(req.begin(), req.end(), "surface_velocity_rms") != req.end();
  if(hasSurfaceVelocityRMS) {
    std::string fname = params->get<std::string>("Surface Velocity RMS File Name", "velocity_RMS.ascii");
    readFileSerial(fname, tempSV, comm);
    velocityRMSVec = Teuchos::rcp(new Tpetra_MultiVector (nodes_map, neq_));
    velocityRMSVec->doImport(tempSV, *importOperator, Tpetra::INSERT);
  }

  if (comm->getRank() == 0) std::cout << " done." << std::endl;

  GO elemColumnShift = (Ordering == 1) ? 1 : maxGlobalElements2D;
  int lElemColumnShift = (Ordering == 1) ? 1 : cells2D.size();
  int elemLayerShift = (Ordering == 0) ? 1 : numLayers;

  GO vertexColumnShift = (Ordering == 1) ? 1 : maxGlobalVertices2dId;
  int lVertexColumnShift = (Ordering == 1) ? 1 : nodes2D.size();
  int vertexLayerShift = (Ordering == 0) ? 1 : numLayers + 1;

  GO edgeColumnShift = (Ordering == 1) ? 1 : maxGlobalEdges2D;
  int lEdgeColumnShift = (Ordering == 1) ? 1 : edges2D.size();
  int edgeLayerShift = (Ordering == 0) ? 1 : numLayers;

  this->layered_mesh_numbering = (Ordering==0) ?
      Teuchos::rcp(new LayeredMeshNumbering<LO>(lVertexColumnShift,Ordering,layerThicknessRatio)):
      Teuchos::rcp(new LayeredMeshNumbering<LO>(vertexLayerShift,Ordering,layerThicknessRatio));

  this->SetupFieldData(comm, neq_, req, sis, worksetSize);

  metaData->commit();

  bulkData->modification_begin(); // Begin modifying the mesh

  stk::mesh::PartVector nodePartVec;
  stk::mesh::PartVector singlePartVec(1);
  stk::mesh::PartVector emptyPartVec;
  unsigned int ebNo = 0; //element block #???

  singlePartVec[0] = nsPartVec["bottom"];

  typedef AbstractSTKFieldContainer::ScalarFieldType ScalarFieldType;
  typedef AbstractSTKFieldContainer::VectorFieldType VectorFieldType;
  typedef AbstractSTKFieldContainer::QPScalarFieldType ElemScalarFieldType;

  AbstractSTKFieldContainer::IntScalarFieldType* proc_rank_field = fieldContainer->getProcRankField();
  VectorFieldType* coordinates_field = fieldContainer->getCoordinatesField();
  stk::mesh::FieldBase const* coordinates_field2d = metaData2D.coordinate_field();
  VectorFieldType* surface_velocity_field = metaData->get_field<VectorFieldType>(stk::topology::NODE_RANK, "surface_velocity");
  VectorFieldType* surface_velocity_RMS_field = metaData->get_field<VectorFieldType>(stk::topology::NODE_RANK, "surface_velocity_rms");
  ScalarFieldType* surface_height_field = metaData->get_field<ScalarFieldType>(stk::topology::NODE_RANK, "surface_height");
  ScalarFieldType* thickness_field = metaData->get_field<ScalarFieldType>(stk::topology::NODE_RANK, "thickness");
  ScalarFieldType* bed_topography_field = metaData->get_field<ScalarFieldType>(stk::topology::NODE_RANK, "bed_topography");
  ScalarFieldType* basal_friction_field = metaData->get_field<ScalarFieldType>(stk::topology::NODE_RANK, "basal_friction");
  ElemScalarFieldType* temperature_field = metaData->get_field<ElemScalarFieldType>(stk::topology::ELEMENT_RANK, "temperature");
  ElemScalarFieldType* flowRate_field = metaData->get_field<ElemScalarFieldType>(stk::topology::ELEMENT_RANK, "flow_factor");

  std::vector<GO> prismMpasIds(NumBaseElemeNodes), prismGlobalIds(2 * NumBaseElemeNodes);

  for (int i = 0; i < (numLayers + 1) * nodes2D.size(); i++) {
    int ib = (Ordering == 0) * (i % lVertexColumnShift) + (Ordering == 1) * (i / vertexLayerShift);
    int il = (Ordering == 0) * (i / lVertexColumnShift) + (Ordering == 1) * (i % vertexLayerShift);
    stk::mesh::Entity node;
    stk::mesh::Entity node2d = nodes2D[ib];
    stk::mesh::EntityId node2dId = bulkData2D.identifier(node2d) - 1;
    if (il == 0)
      node = bulkData->declare_entity(stk::topology::NODE_RANK, il * vertexColumnShift + vertexLayerShift * node2dId + 1, singlePartVec);
    else
      node = bulkData->declare_entity(stk::topology::NODE_RANK, il * vertexColumnShift + vertexLayerShift * node2dId + 1, nodePartVec);

    std::vector<int> sharing_procs;
    bulkData2D.comm_shared_procs( bulkData2D.entity_key(node2d), sharing_procs );
    for(int iproc=0; iproc<sharing_procs.size(); ++iproc)
      bulkData->add_node_sharing(node, sharing_procs[iproc]);


    double* coord = stk::mesh::field_data(*coordinates_field, node);
    double const* coord2d = (double const*) stk::mesh::field_data(*coordinates_field2d, node2d);
    coord[0] = coord2d[0];
    coord[1] = coord2d[1];

    int lid = nodes_map->getLocalElement((GO)(node2dId));
    coord[2] = sHeightVec_constView[lid] - thickVec_constView[lid] * (1. - levelsNormalizedThickness[il]);

    if(hasSurface_height && surface_height_field) {
      double* sHeight = stk::mesh::field_data(*surface_height_field, node);
      sHeight[0] = sHeightVec_constView[lid];
    }

    if(hasThickness && thickness_field) {
      double* thick = stk::mesh::field_data(*thickness_field, node);
      thick[0] = thickVec_constView[lid];
    }

    if(hasBed_topography && bed_topography_field) {
      double* bedTopo = stk::mesh::field_data(*bed_topography_field, node);
      bedTopo[0] = bTopographyVec_constView[lid];
    }

    if(surface_velocity_field) {
      double* sVelocity = stk::mesh::field_data(*surface_velocity_field, node);
      Teuchos::ArrayRCP<const ST> sVelocityVec_constView_0 = sVelocityVec->getVectorNonConst(0)->get1dView();
      Teuchos::ArrayRCP<const ST> sVelocityVec_constView_1 = sVelocityVec->getVectorNonConst(1)->get1dView();
      sVelocity[0] = sVelocityVec_constView_0[lid];
      sVelocity[1] = sVelocityVec_constView_1[lid];
    }

    if(surface_velocity_RMS_field) {
      double* velocityRMS = stk::mesh::field_data(*surface_velocity_RMS_field, node);
      Teuchos::ArrayRCP<const ST> velocityRMSVec_constView_0 = velocityRMSVec->getVectorNonConst(0)->get1dView();
      Teuchos::ArrayRCP<const ST> velocityRMSVec_constView_1 = velocityRMSVec->getVectorNonConst(1)->get1dView();
      velocityRMS[0] = velocityRMSVec_constView_0[lid];
      velocityRMS[1] = velocityRMSVec_constView_1[lid];

    }

    if(hasBasal_friction && basal_friction_field) {
      double* bFriction = stk::mesh::field_data(*basal_friction_field, node);
      bFriction[0] = bFrictionVec->get1dView()[lid];
    }
  }

  GO tetrasLocalIdsOnPrism[3][4];

  for (int i = 0; i < cells2D.size() * numLayers; i++) {

    int ib = (Ordering == 0) * (i % lElemColumnShift) + (Ordering == 1) * (i / elemLayerShift);
    int il = (Ordering == 0) * (i / lElemColumnShift) + (Ordering == 1) * (i % elemLayerShift);

    int shift = il * vertexColumnShift;

    singlePartVec[0] = partVec[ebNo];

    //TODO: this could be done only in the first layer and then copied into the other layers

    stk::mesh::Entity const* rel = bulkData2D.begin_nodes(cells2D[ib]);
    double tempOnPrism = 0; //Set temperature constant on each prism/Hexa
    Teuchos::ArrayRCP<const ST> temperatureVecInterp_constView_il,  temperatureVecInterp_constView_ilplus1;
    if(hasTemperature) {
      temperatureVecInterp_constView_il = temperatureVecInterp->getVectorNonConst(il)->get1dView();
      temperatureVecInterp_constView_ilplus1 = temperatureVecInterp->getVectorNonConst(il + 1)->get1dView();
    }

    double flowRateOnPrism = 0; //Set temperature constant on each prism/Hexa
    Teuchos::ArrayRCP<const ST> flowRateVecInterp_constView_il, flowRateVecInterp_constView_ilplus1;
    if(hasFlowRate) {
      flowRateVecInterp_constView_il = flowRateVecInterp->getVectorNonConst(il)->get1dView();
    }
    for (int j = 0; j < NumBaseElemeNodes; j++) {
      stk::mesh::EntityId node2dId = bulkData2D.identifier(rel[j]) - 1;
      int node2dLId = nodes_map->getLocalElement((GO)node2dId);
      stk::mesh::EntityId mpasLowerId = vertexLayerShift * node2dId;
      stk::mesh::EntityId lowerId = shift + vertexLayerShift * node2dId;
      prismMpasIds[j] = mpasLowerId;
      prismGlobalIds[j] = lowerId;
      prismGlobalIds[j + NumBaseElemeNodes] = lowerId + vertexColumnShift;
      if(hasTemperature)
        tempOnPrism += 1. / NumBaseElemeNodes / 2. * (temperatureVecInterp_constView_il[node2dLId] + temperatureVecInterp_constView_ilplus1[node2dLId]);
      if(hasFlowRate)
        flowRateOnPrism += 1. / NumBaseElemeNodes  * flowRateVecInterp_constView_il[node2dLId];
    }

    switch (ElemShape) {
    case Tetrahedron: {
      tetrasFromPrismStructured(&prismMpasIds[0], &prismGlobalIds[0], tetrasLocalIdsOnPrism);

      stk::mesh::EntityId prismId = il * elemColumnShift + elemLayerShift * (bulkData2D.identifier(cells2D[ib]) - 1);
      for (int iTetra = 0; iTetra < 3; iTetra++) {
        stk::mesh::Entity elem = bulkData->declare_entity(stk::topology::ELEMENT_RANK, 3 * prismId + iTetra + 1, singlePartVec);
        for (int j = 0; j < 4; j++) {
          stk::mesh::Entity node = bulkData->get_entity(stk::topology::NODE_RANK, tetrasLocalIdsOnPrism[iTetra][j] + 1);
          bulkData->declare_relation(elem, node, j);
        }
        int* p_rank = (int*) stk::mesh::field_data(*proc_rank_field, elem);
        p_rank[0] = comm->getRank();
        if(hasFlowRate && flowRate_field) {
          double* flowRate = stk::mesh::field_data(*flowRate_field, elem);
          flowRate[0] = flowRateOnPrism;
        }
        if(hasTemperature && temperature_field) {
          double* temperature = stk::mesh::field_data(*temperature_field, elem);
          temperature[0] = tempOnPrism;
        }
      }
    }
      break;
    case Wedge:
    case Hexahedron: {
      stk::mesh::EntityId prismId = il * elemColumnShift + elemLayerShift * (bulkData2D.identifier(cells2D[ib]) - 1);
      stk::mesh::Entity elem = bulkData->declare_entity(stk::topology::ELEMENT_RANK, prismId + 1, singlePartVec);
      for (int j = 0; j < 2 * NumBaseElemeNodes; j++) {
        stk::mesh::Entity node = bulkData->get_entity(stk::topology::NODE_RANK, prismGlobalIds[j] + 1);
        bulkData->declare_relation(elem, node, j);
      }
      int* p_rank = (int*) stk::mesh::field_data(*proc_rank_field, elem);
      p_rank[0] = comm->getRank();
      if(hasFlowRate && flowRate_field) {
        double* flowRate = stk::mesh::field_data(*flowRate_field, elem);
        flowRate[0] = flowRateOnPrism;
      }
      if(hasTemperature && temperature_field) {
        double* temperature = stk::mesh::field_data(*temperature_field, elem);
        temperature[0] = tempOnPrism;
      }
    }
    }
  }

  int numSubelemOnPrism, numBasalSidePoints;
  int basalSideLID, upperSideLID;

  switch (ElemShape) {
  case Tetrahedron:
    numSubelemOnPrism = 3;
    numBasalSidePoints = 3;
    basalSideLID = 3;  //depends on how the tetrahedron is located in the Prism, see tetraFaceIdOnPrismFaceId below.
    upperSideLID = 1;
    break;
  case Wedge:
    numSubelemOnPrism = 1;
    numBasalSidePoints = 3;
    basalSideLID = 3;  //depends on how the tetrahedron is located in the Prism.
    upperSideLID = 4;
    break;
  case Hexahedron:
    numSubelemOnPrism = 1;
    numBasalSidePoints = 4;
    basalSideLID = 4;  //depends on how the tetrahedron is located in the Prism.
    upperSideLID = 5;
    break;
  }

  singlePartVec[0] = ssPartVec["lateralside"];

  //The following two arrays have being computed offline using the computeMap function in .hpp file.

  //tetraFaceIdOnPrismFaceId[ minIndex ][ PrismFaceID ]
  int tetraFaceIdOnPrismFaceId[6][5] = {{0, 1, 2, 3, 1}, {2, 0, 1, 3, 1}, {1, 2, 0, 3, 1}, {2, 1, 0, 1 ,3}, {0, 2, 1, 1, 3}, {1, 0, 2, 1, 3}};

  //tetraAdjacentToPrismLateralFace[ minIndex ][ prismType ][ PrismFaceID ][ iTetra ]. There are to Terahedra adjacent to a Prism face. iTetra in {0,1}
  int tetraAdjacentToPrismLateralFace[6][2][3][2] = { { { { 1, 2 }, { 0, 1 }, { 0, 2 } }, { { 0, 2 }, { 0, 1 }, { 1, 2 } } },
                                                      { { { 0, 2 }, { 1, 2 }, { 0, 1 } }, { { 1, 2 }, { 0, 2 }, { 0, 1 } } },
                                                      { { { 0, 1 }, { 0, 2 }, { 1, 2 } }, { { 0, 1 }, { 1, 2 }, { 0, 2 } } },
                                                      { { { 0, 2 }, { 0, 1 }, { 1, 2 } }, { { 1, 2 }, { 0, 1 }, { 0, 2 } } },
                                                      { { { 1, 2 }, { 0, 2 }, { 0, 1 } }, { { 0, 2 }, { 1, 2 }, { 0, 1 } } },
                                                      { { { 0, 1 }, { 1, 2 }, { 0, 2 } }, { { 0, 1 }, { 0, 2 }, { 1, 2 } } } };

  for (int i = 0; i < edges2D.size() * numLayers; i++) {
    int ib = (Ordering == 0) * (i % lEdgeColumnShift) + (Ordering == 1) * (i / edgeLayerShift);
    // if(!isBoundaryEdge[ib]) continue; //WARNING: assuming that all the edges stored are boundary edges!!

    stk::mesh::Entity edge2d = edges2D[ib];
    stk::mesh::Entity const* rel = bulkData2D.begin_elements(edge2d);
    stk::mesh::ConnectivityOrdinal const* ordinals = bulkData2D.begin_element_ordinals(edge2d);

    int il = (Ordering == 0) * (i / lEdgeColumnShift) + (Ordering == 1) * (i % edgeLayerShift);
    stk::mesh::Entity elem2d = rel[0];
    stk::mesh::EntityId edgeLID = ordinals[0]; //bulkData2D.identifier(rel[0]);

    stk::mesh::EntityId basalElemId = bulkData2D.identifier(elem2d) - 1;
    stk::mesh::EntityId Edge2dId = bulkData2D.identifier(edge2d) - 1;
    switch (ElemShape) {
    case Tetrahedron: {
      rel = bulkData2D.begin_nodes(elem2d);
      for (int j = 0; j < NumBaseElemeNodes; j++) {
        stk::mesh::EntityId node2dId = bulkData2D.identifier(rel[j]) - 1;
        prismMpasIds[j] = vertexLayerShift * node2dId;
      }
      int minIndex;
      int pType = prismType(&prismMpasIds[0], minIndex);
      stk::mesh::EntityId tetraId = 3 * il * elemColumnShift + 3 * elemLayerShift * basalElemId;

      stk::mesh::Entity elem0 = bulkData->get_entity(stk::topology::ELEMENT_RANK, tetraId + tetraAdjacentToPrismLateralFace[minIndex][pType][edgeLID][0] + 1);
      stk::mesh::Entity elem1 = bulkData->get_entity(stk::topology::ELEMENT_RANK, tetraId + tetraAdjacentToPrismLateralFace[minIndex][pType][edgeLID][1] + 1);

      stk::mesh::Entity side0 = bulkData->declare_entity(metaData->side_rank(), 2 * edgeColumnShift * il +  2 * Edge2dId * edgeLayerShift + 1, singlePartVec);
      stk::mesh::Entity side1 = bulkData->declare_entity(metaData->side_rank(), 2 * edgeColumnShift * il +  2 * Edge2dId * edgeLayerShift + 1 + 1, singlePartVec);

      bulkData->declare_relation(elem0, side0, tetraFaceIdOnPrismFaceId[minIndex][edgeLID]);
      bulkData->declare_relation(elem1, side1, tetraFaceIdOnPrismFaceId[minIndex][edgeLID]);

      stk::mesh::Entity const* rel_elemNodes0 = bulkData->begin_nodes(elem0);
      stk::mesh::Entity const* rel_elemNodes1 = bulkData->begin_nodes(elem1);
      for (int j = 0; j < 3; j++) {
     //   std::cout << j <<", " << edgeLID << ", " << minIndex << ", " << tetraFaceIdOnPrismFaceId[minIndex][edgeLID] << ","  << std::endl;
        stk::mesh::Entity node0 = rel_elemNodes0[this->meshSpecs[0]->ctd.side[tetraFaceIdOnPrismFaceId[minIndex][edgeLID]].node[j]];
        bulkData->declare_relation(side0, node0, j);
        stk::mesh::Entity node1 = rel_elemNodes1[this->meshSpecs[0]->ctd.side[tetraFaceIdOnPrismFaceId[minIndex][edgeLID]].node[j]];
        bulkData->declare_relation(side1, node1, j);
      }
    }

      break;
    case Wedge:
    case Hexahedron: {
      stk::mesh::EntityId prismId = il * elemColumnShift + elemLayerShift * basalElemId;
      stk::mesh::Entity elem = bulkData->get_entity(stk::topology::ELEMENT_RANK, prismId + 1);
      stk::mesh::Entity side = bulkData->declare_entity(metaData->side_rank(), edgeColumnShift * il +Edge2dId * edgeLayerShift + 1, singlePartVec);
      bulkData->declare_relation(elem, side, edgeLID);

      stk::mesh::Entity const* rel_elemNodes = bulkData->begin_nodes(elem);
      for (int j = 0; j < 4; j++) {
        stk::mesh::Entity node = rel_elemNodes[this->meshSpecs[0]->ctd.side[edgeLID].node[j]];
        bulkData->declare_relation(side, node, j);
      }
    }
    break;
    }
  }

  //then we store the lower and upper faces of prisms, which corresponds to triangles of the basal mesh
  edgeLayerShift = (Ordering == 0) ? 1 : numLayers + 1;
  edgeColumnShift = elemColumnShift;

  singlePartVec[0] = ssPartVec["basalside"];


  GO edgeOffset = maxGlobalEdges2D * numLayers;
  if(ElemShape == Tetrahedron) edgeOffset *= 2;

  for (int i = 0; i < cells2D.size(); i++) {
    stk::mesh::Entity elem2d = cells2D[i];
    stk::mesh::EntityId elem2d_id = bulkData2D.identifier(elem2d) - 1;
    stk::mesh::Entity side = bulkData->declare_entity(metaData->side_rank(), elem2d_id + edgeOffset + 1, singlePartVec);
    stk::mesh::Entity elem = bulkData->get_entity(stk::topology::ELEMENT_RANK, elem2d_id * numSubelemOnPrism * elemLayerShift + 1);
    bulkData->declare_relation(elem, side, basalSideLID);

    stk::mesh::Entity const* rel_elemNodes = bulkData->begin_nodes(elem);
    for (int j = 0; j < numBasalSidePoints; j++) {
      stk::mesh::Entity node = rel_elemNodes[this->meshSpecs[0]->ctd.side[basalSideLID].node[j]];
      bulkData->declare_relation(side, node, j);
    }
  }

  singlePartVec[0] = ssPartVec["upperside"];

  edgeOffset += maxGlobalElements2D;

  for (int i = 0; i < cells2D.size(); i++) {
    stk::mesh::Entity elem2d = cells2D[i];
    stk::mesh::EntityId elem2d_id = bulkData2D.identifier(elem2d) - 1;
    stk::mesh::Entity side = bulkData->declare_entity(metaData->side_rank(), elem2d_id  + edgeOffset + 1, singlePartVec);
    stk::mesh::Entity elem = bulkData->get_entity(stk::topology::ELEMENT_RANK, elem2d_id * numSubelemOnPrism * elemLayerShift + (numLayers - 1) * numSubelemOnPrism * elemColumnShift + 1 + (numSubelemOnPrism - 1));
    bulkData->declare_relation(elem, side, upperSideLID);

    stk::mesh::Entity const* rel_elemNodes = bulkData->begin_nodes(elem);
    for (int j = 0; j < numBasalSidePoints; j++) {
      stk::mesh::Entity node = rel_elemNodes[this->meshSpecs[0]->ctd.side[upperSideLID].node[j]];
      bulkData->declare_relation(side, node, j);
    }
  }
  //Albany::fix_node_sharing(*bulkData);
  bulkData->modification_end();

  if (params->get("Export 2D Data",false))
  {
    // We export the basal mesh in GMSH format
    std::ofstream ofile;
    ofile.open (params->get("GMSH 2D Output File Name","basal_mesh.msh"));
    TEUCHOS_TEST_FOR_EXCEPTION (!ofile.is_open(), std::logic_error, "Error! Cannot open file 'basal_mesh.msh'.\n");

    // Preamble
    ofile << "$MeshFormat\n"
          << "2.2 0 8\n"
          << "$EndMeshFormat\n";

    // Nodes
    ofile << "$Nodes\n" << nodes2D.size() << "\n";
    stk::mesh::Entity node2d;
    stk::mesh::EntityId nodeId;
    for (int i(0); i<nodes2D.size(); ++i)
    {
      node2d = bulkData2D.get_entity(stk::topology::NODE_RANK, i + 1);
      nodeId = bulkData2D.identifier(nodes2D[i]);

      double const* coord2d = (double const*) stk::mesh::field_data(*coordinates_field2d, node2d);

      ofile << nodeId << " " << coord2d[0] << " " << coord2d[1] << " " << 0. << "\n";
    }
    ofile << "$EndNodes\n";

    // Mesh Elements (including edges)
    ofile << "$Elements\n";
    ofile << edges2D.size()+cells2D.size() << "\n";

    int counter = 1;

    // edges
    for (int i(0); i<edges2D.size(); ++i)
    {
      stk::mesh::Entity const* rel = bulkData2D.begin_nodes(edges2D[i]);

      ofile << counter << " " << 1 << " " << 2 << " " << 30 << " " << 1;
      for (int j(0); j<2; ++j)
      {
        stk::mesh::EntityId node2dId = bulkData2D.identifier(rel[j]);
        ofile << " " << node2dId;
      }
      ofile << "\n";
      ++counter;
    }

    // elements
    for (int i(0); i<cells2D.size(); ++i)
    {
      stk::mesh::Entity const* rel = bulkData2D.begin_nodes(cells2D[i]);
      ofile << counter << " " << 2 << " " << 2 << " " << 100 << " " << 11;
      for (int j(0); j<3; ++j)
      {
        stk::mesh::EntityId node2dId = bulkData2D.identifier(rel[j]);
        ofile << " " << node2dId;
      }
      ofile << "\n";
      ++counter;
    }
    ofile << "$EndElements\n";

    ofile.close();
  }
}
Albany::OrdinarySTKFieldContainer<Interleaved>::OrdinarySTKFieldContainer(
  const Teuchos::RCP<Teuchos::ParameterList>& params_,
  const Teuchos::RCP<stk::mesh::MetaData>& metaData_,
  const int neq_,
  const AbstractFieldContainer::FieldContainerRequirements& req,
  const int numDim_,
  const Teuchos::RCP<Albany::StateInfoStruct>& sis)
  : GenericSTKFieldContainer<Interleaved>(params_, metaData_, neq_, numDim_),
      buildSphereVolume(false) {

  typedef typename AbstractSTKFieldContainer::VectorFieldType VFT;
  typedef typename AbstractSTKFieldContainer::ScalarFieldType SFT;

#ifdef ALBANY_LCM
  buildSphereVolume = (std::find(req.begin(), req.end(), "Sphere Volume") != req.end());
#endif

  //Start STK stuff
  this->coordinates_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, "coordinates");
  solution_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                                     params_->get<std::string>("Exodus Solution Name", "solution"));

#ifdef ALBANY_LCM
  residual_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                                     params_->get<std::string>("Exodus Residual Name", "residual"));
#endif

  stk::mesh::put_field(*this->coordinates_field , metaData_->universal_part(), numDim_);
  stk::mesh::put_field(*solution_field , metaData_->universal_part(), neq_);

#ifdef ALBANY_LCM
  stk::mesh::put_field(*residual_field , metaData_->universal_part() , neq_);
#endif

#ifdef ALBANY_SEACAS
  stk::io::set_field_role(*this->coordinates_field, Ioss::Field::MESH);
  stk::io::set_field_role(*solution_field, Ioss::Field::TRANSIENT);
#ifdef ALBANY_LCM
  stk::io::set_field_role(*residual_field, Ioss::Field::TRANSIENT);
#endif

#endif

#ifdef ALBANY_LCM
  // sphere volume is a mesh attribute read from a genesis mesh file containing sphere element (used for peridynamics)
  if(buildSphereVolume){
    this->sphereVolume_field = metaData_->get_field< stk::mesh::Field<double> >(stk::topology::ELEMENT_RANK, "volume");
    TEUCHOS_TEST_FOR_EXCEPTION(this->sphereVolume_field == 0, std::logic_error, "\n**** Error:  Expected volume field for sphere elements, field not found.\n");
    stk::io::set_field_role(*this->sphereVolume_field, Ioss::Field::ATTRIBUTE);
  }
#endif

  // If the problem requests that the initial guess at the solution equals the input node coordinates,
  // set that here
  /*
    if(std::find(req.begin(), req.end(), "Initial Guess Coords") != req.end()){
       this->copySTKField(this->coordinates_field, solution_field);
    }
  */

  this->buildStateStructs(sis);

  initializeSTKAdaptation();
}
Albany::OrdinarySTKFieldContainer<Interleaved>::OrdinarySTKFieldContainer(
  const Teuchos::RCP<Teuchos::ParameterList>& params_,
  const Teuchos::RCP<stk::mesh::MetaData>& metaData_,
  const int neq_,
  const AbstractFieldContainer::FieldContainerRequirements& req,
  const int numDim_,
  const Teuchos::RCP<Albany::StateInfoStruct>& sis)
  : GenericSTKFieldContainer<Interleaved>(params_, metaData_, neq_, numDim_),
      buildSphereVolume(false) {

  typedef typename AbstractSTKFieldContainer::VectorFieldType VFT;
  typedef typename AbstractSTKFieldContainer::ScalarFieldType SFT;

  //Start STK stuff
  this->coordinates_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, "coordinates");
  solution_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                                     params_->get<std::string>("Exodus Solution Name", "solution"));

#if defined(ALBANY_LCM)
  residual_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK,
                                                     params_->get<std::string>("Exodus Residual Name", "residual"));
#endif

  stk::mesh::put_field(*this->coordinates_field , metaData_->universal_part(), numDim_);
  stk::mesh::put_field(*solution_field , metaData_->universal_part(), neq_);

#if defined(ALBANY_LCM)
  stk::mesh::put_field(*residual_field , metaData_->universal_part() , neq_);
#endif

#ifdef ALBANY_SEACAS
  stk::io::set_field_role(*this->coordinates_field, Ioss::Field::MESH);
  stk::io::set_field_role(*solution_field, Ioss::Field::TRANSIENT);
#if defined(ALBANY_LCM)
  stk::io::set_field_role(*residual_field, Ioss::Field::TRANSIENT);
#endif

#endif

#if defined(ALBANY_LCM)
  // sphere volume is a mesh attribute read from a genesis mesh file containing sphere element (used for peridynamics)
  // for whatever reason, its type is stk::mesh::Field<double, stk::mesh::Cartesian3d>
  // the read won't work if you try to read it as a SFT
  bool hasSphereVolumeFieldContainerRequirement = (std::find(req.begin(), req.end(), "Sphere Volume") != req.end());
  if(hasSphereVolumeFieldContainerRequirement){
    this->sphereVolume_field = metaData_->template get_field< stk::mesh::Field<double, stk::mesh::Cartesian3d> >(stk::topology::ELEMENT_RANK, "volume");
    if(this->sphereVolume_field != 0){
      buildSphereVolume = true;
      stk::io::set_field_role(*this->sphereVolume_field, Ioss::Field::ATTRIBUTE);
    }
  }
#endif

  // If the problem requests that the initial guess at the solution equals the input node coordinates,
  // set that here
  /*
    if(std::find(req.begin(), req.end(), "Initial Guess Coords") != req.end()){
       this->copySTKField(this->coordinates_field, solution_field);
    }
  */

  this->buildStateStructs(sis);

  initializeSTKAdaptation();
}
Пример #5
0
void
Albany::IossSTKMeshStruct::setFieldAndBulkData (
          const Teuchos::RCP<const Teuchos_Comm>& commT,
          const Teuchos::RCP<Teuchos::ParameterList>& params,
          const unsigned int neq_,
          const AbstractFieldContainer::FieldContainerRequirements& req,
          const Teuchos::RCP<Albany::StateInfoStruct>& sis,
          const unsigned int worksetSize)
{
  this->SetupFieldData(commT, neq_, req, sis, worksetSize);

  mesh_data->set_bulk_data(*bulkData);

  *out << "IOSS-STK: number of node sets = " << nsPartVec.size() << std::endl;
  *out << "IOSS-STK: number of side sets = " << ssPartVec.size() << std::endl;

  mesh_data->add_all_mesh_fields_as_input_fields();
  std::vector<stk::io::MeshField> missing;

  metaData->commit();

  // Restart index to read solution from exodus file.
  int index = params->get("Restart Index",-1); // Default to no restart
  double res_time = params->get<double>("Restart Time",-1.0); // Default to no restart
  Ioss::Region& region = *(mesh_data->get_input_io_region());

  /*
   * The following code block reads a single mesh on PE 0, then distributes the mesh across
   * the other processors. stk_rebalance is used, which requires Zoltan
   *
   * This code is only compiled if ALBANY_MPI and ALBANY_ZOLTAN are true
   */

#ifdef ALBANY_ZOLTAN // rebalance needs Zoltan

  if(useSerialMesh){

    // trick to avoid hanging
    bulkData->modification_begin();

    if(commT->getRank() == 0){ // read in the mesh on PE 0


      //stk::io::process_mesh_bulk_data(region, *bulkData);
      mesh_data->populate_bulk_data();
      //bulkData = &mesh_data->bulk_data();

      // Read solution from exodus file.
      if (index >= 0) { // User has specified a time step to restart at
        *out << "Restart Index set, reading solution index : " << index << std::endl;
        mesh_data->read_defined_input_fields(index, &missing);
        m_restartDataTime = region.get_state_time(index);
        m_hasRestartSolution = true;
      }
      else if (res_time >= 0) { // User has specified a time to restart at
        *out << "Restart solution time set, reading solution time : " << res_time << std::endl;
        mesh_data->read_defined_input_fields(res_time, &missing);
        m_restartDataTime = res_time;
        m_hasRestartSolution = true;
      }
      else {

        *out << "Neither restart index or time are set. Not reading solution data from exodus file"<< std::endl;

      }
    }
    else {
      // trick to avoid hanging
      bulkData->modification_begin(); bulkData->modification_begin();
    }

    bulkData->modification_end();

  } // End UseSerialMesh - reading mesh on PE 0

  else
#endif

    /*
     * The following code block reads a single mesh when Albany is compiled serially, or a
     * Nemspread fileset if ALBANY_MPI is true.
     *
     */

  { // running in Serial or Parallel read from Nemspread files
    mesh_data->populate_bulk_data();
    if (!usePamgen)
    {
      // Read solution from exodus file.
      if (index >= 0)
      { // User has specified a time step to restart at
        *out << "Restart Index set, reading solution index : " << index << std::endl;
        mesh_data->read_defined_input_fields(index, &missing);
        m_restartDataTime = region.get_state_time(index);
        m_hasRestartSolution = true;
      }
      else if (res_time >= 0)
      { // User has specified a time to restart at
        *out << "Restart solution time set, reading solution time : " << res_time << std::endl;
        mesh_data->read_defined_input_fields(res_time, &missing);
        m_restartDataTime = res_time;
        m_hasRestartSolution = true;
      }
      else
      {
        *out << "Neither restart index or time are set. We still read defined fields in case they are needed (e.g., parameters)."<< std::endl;
//        *out << "Restart Index not set. Not reading solution from exodus (" << index << ")"<< std::endl;
      }
    }

    bulkData->modification_end();

  } // End Parallel Read - or running in serial

  if(m_hasRestartSolution){

    Teuchos::Array<std::string> default_field;
    default_field.push_back("solution");
    Teuchos::Array<std::string> restart_fields =
      params->get<Teuchos::Array<std::string> >("Restart Fields", default_field);

    // Get the fields to be used for restart

    // See what state data was initialized from the stk::io request
    // This should be propagated into stk::io
    const Ioss::ElementBlockContainer& elem_blocks = region.get_element_blocks();

    /*
    // Uncomment to print what fields are in the exodus file
    Ioss::NameList exo_fld_names;
    elem_blocks[0]->field_describe(&exo_fld_names);
    for(std::size_t i = 0; i < exo_fld_names.size(); i++){
    *out << "Found field \"" << exo_fld_names[i] << "\" in exodus file" << std::endl; } */

    for (std::size_t i=0; i<sis->size(); i++) { Albany::StateStruct& st = *((*sis)[i]);
      if(elem_blocks[0]->field_exists(st.name))

        for(std::size_t j = 0; j < restart_fields.size(); j++)

          if(boost::iequals(st.name, restart_fields[j])){

            *out << "Restarting from field \"" << st.name << "\" found in exodus file." << std::endl;
            st.restartDataAvailable = true;
            break;

          }
    }
  }

#ifdef ALBANY_FELIX
  // Load required fields
  stk::mesh::Selector select_owned_in_part = stk::mesh::Selector(metaData->universal_part()) & stk::mesh::Selector(metaData->locally_owned_part());

  stk::mesh::Selector select_overlap_in_part = stk::mesh::Selector(metaData->universal_part()) & (stk::mesh::Selector(metaData->locally_owned_part()) | stk::mesh::Selector(metaData->globally_shared_part()));

  std::vector<stk::mesh::Entity> nodes;
  stk::mesh::get_selected_entities(select_overlap_in_part, bulkData->buckets(stk::topology::NODE_RANK), nodes);

  std::vector<stk::mesh::Entity> elems;
  stk::mesh::get_selected_entities(select_owned_in_part, bulkData->buckets(stk::topology::ELEM_RANK), elems);

  GO numOwnedNodes(0);
  GO numOwnedElems(0);
  numOwnedNodes = stk::mesh::count_selected_entities(select_owned_in_part, bulkData->buckets(stk::topology::NODE_RANK));
  numOwnedElems = stk::mesh::count_selected_entities(select_owned_in_part, bulkData->buckets(stk::topology::ELEM_RANK));

  GO numGlobalVertices = 0;
  GO numGlobalElements = 0;
  Teuchos::reduceAll<int, GO>(*commT, Teuchos::REDUCE_SUM, 1, &numOwnedNodes, &numGlobalVertices);
  Teuchos::reduceAll<int, GO>(*commT, Teuchos::REDUCE_SUM, 1, &numOwnedElems, &numGlobalElements);

  if (commT->getRank() == 0)
  {
    *out << "Checking if requirements are already stored in the mesh. If not, we import them from ascii files.\n";
  }

  Teuchos::Array<GO> nodeIndices(nodes.size()), elemIndices(elems.size());
  for (int i = 0; i < nodes.size(); ++i)
    nodeIndices[i] = bulkData->identifier(nodes[i]) - 1;
  for (int i = 0; i < elems.size(); ++i)
    elemIndices[i] = bulkData->identifier(elems[i]) - 1;


  // Creating the serial and parallel node maps
  const Tpetra::global_size_t INVALID = Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid ();

  Teuchos::RCP<const Tpetra_Map> nodes_map = Tpetra::createNonContigMapWithNode<LO, GO> (nodeIndices, commT, KokkosClassic::Details::getNode<KokkosNode>());
  Teuchos::RCP<const Tpetra_Map> elems_map = Tpetra::createNonContigMapWithNode<LO, GO> (elemIndices, commT, KokkosClassic::Details::getNode<KokkosNode>());

  int numMyNodes = (commT->getRank() == 0) ? numGlobalVertices : 0;
  int numMyElements = (commT->getRank() == 0) ? numGlobalElements : 0;
  Teuchos::RCP<const Tpetra_Map> serial_nodes_map = Teuchos::rcp(new const Tpetra_Map(INVALID, numMyNodes, 0, commT));
  Teuchos::RCP<const Tpetra_Map> serial_elems_map = Teuchos::rcp(new const Tpetra_Map(INVALID, numMyElements, 0, commT));

  // Creating the Tpetra_Import object (to transfer from serial to parallel vectors)
  Tpetra_Import importOperatorNode (serial_nodes_map, nodes_map);
  Tpetra_Import importOperatorElem (serial_elems_map, elems_map);

  Teuchos::ParameterList* req_fields_info;
  if (params->isSublist("Required Fields Info"))
  {
    req_fields_info = &params->sublist("Required Fields Info");

    for (AbstractFieldContainer::FieldContainerRequirements::const_iterator it=req.begin(); it!=req.end(); ++it)
    {
      // Get the file name
      std::string temp_str = *it + " File Name";
      std::string fname = req_fields_info->get<std::string>(temp_str,"");

      // Ge the file type (if not specified, assume Scalar)
      temp_str = *it + " Field Type";
      std::string ftype = req_fields_info->get<std::string>(temp_str,"");
      if (ftype=="")
      {
        *out << "Warning! No field type specified for field " << *it << ". We skip it and hope is already present in the mesh...\n";
        continue;
      }


      stk::mesh::Entity node, elem;
      stk::mesh::EntityId nodeId, elemId;
      int lid;
      double* values;

      typedef AbstractSTKFieldContainer::QPScalarFieldType  QPScalarFieldType;
      typedef AbstractSTKFieldContainer::QPVectorFieldType  QPVectorFieldType;
      typedef AbstractSTKFieldContainer::ScalarFieldType    ScalarFieldType;
      typedef AbstractSTKFieldContainer::VectorFieldType    VectorFieldType;

      // Depending on the field type, we need to use different pointers
      if (ftype == "Node Scalar")
      {
        // Creating the serial and (possibly) parallel Tpetra service vectors
        Tpetra_Vector serial_req_vec(serial_nodes_map);
        Tpetra_Vector req_vec(nodes_map);

        temp_str = *it + " Value";
        if (req_fields_info->isParameter(temp_str))
        {
          *out << "Discarding other info about Node Scalar field " << *it << " and filling it with constant value " << req_fields_info->get<double>(temp_str) << "\n";
          // For debug, we allow to fill the field with a given uniform value
          fillTpetraVec (serial_req_vec,req_fields_info->get<double>(temp_str));
        }
        else if (fname!="")
        {
          *out << "Reading Node Scalar field " << *it << " from file " << fname << "\n";
          // Read the input file and stuff it in the Tpetra vector
          readScalarFileSerial (fname,serial_req_vec,commT);

          temp_str = *it + " Scale Factor";
          if (req_fields_info->isParameter(temp_str))
          {
            double scale_factor = req_fields_info->get<double>(temp_str);
            serial_req_vec.scale (scale_factor);
          }
        }
        else
        {
          bool found = false;
          for (int i(0); i<missing.size(); ++i)
          {
            if (missing[i].field()->name()==*it)
            {
              *out << "No file name nor constant value specified for Node Scalar field " << *it << "; initializing it to 0.\n";
              fillTpetraVec (serial_req_vec,0.);
              found = true;
              break;
            }
          }
          if (!found)
          {
            *out << "Using mesh-stored values for Node Scalar field " << *it << " since no constant value nor filename has been specified\n";
            continue;
          }
        }

        // Fill the (possibly) parallel vector
        req_vec.doImport(serial_req_vec,importOperatorNode,Tpetra::INSERT);

        // Extracting the mesh field and the tpetra vector view
        ScalarFieldType* field = metaData->get_field<ScalarFieldType>(stk::topology::NODE_RANK, *it);

        TEUCHOS_TEST_FOR_EXCEPTION (field==0, std::logic_error, "Error! Field " << *it << " not present (perhaps is 'Elem Scalar'?).\n");

        Teuchos::ArrayRCP<const ST> req_vec_view = req_vec.get1dView();

        //Now we have to stuff the vector in the mesh data
        for (int i(0); i<nodes.size(); ++i)
        {
          nodeId = bulkData->identifier(nodes[i]) - 1;
          lid    = nodes_map->getLocalElement((GO)(nodeId));

          values = stk::mesh::field_data(*field, nodes[i]);
          values[0] = req_vec_view[lid];
        }
      }
      else if (ftype == "Elem Scalar")
      {
        // Creating the serial and (possibly) parallel Tpetra service vectors
        Tpetra_Vector serial_req_vec(serial_elems_map);
        Tpetra_Vector req_vec(elems_map);

        temp_str = *it + " Value";
        if (req_fields_info->isParameter(temp_str))
        {
          *out << "Discarding other info about Elem Scalar field " << *it << " and filling it with constant value " << req_fields_info->get<double>(temp_str) << "\n";
          // For debug, we allow to fill the field with a given uniform value
          fillTpetraVec (serial_req_vec,req_fields_info->get<double>(temp_str));
        }
        else if (fname!="")
        {
          *out << "Reading Elem Scalar field " << *it << " from file " << fname << "\n";
          // Read the input file and stuff it in the Tpetra vector
          readScalarFileSerial (fname,serial_req_vec,commT);

          temp_str = *it + " Scale Factor";
          if (req_fields_info->isParameter(temp_str))
          {
            double scale_factor = req_fields_info->get<double>(temp_str);
            serial_req_vec.scale (scale_factor);
          }
        }
        else
        {
          bool found = false;
          for (int i(0); i<missing.size(); ++i)
          {
            if (missing[i].field()->name()==*it)
            {
              *out << "No file name nor constant value specified for Elem Scalar field " << *it << "; initializing it to 0.\n";
              fillTpetraVec (serial_req_vec,0.);
              found = true;
              break;
            }
          }
          if (!found)
          {
            *out << "Using mesh-stored values for Elem Scalar field " << *it << " since no constant value nor filename has been specified\n";
            continue;
          }
        }

        // Fill the (possibly) parallel vector
        req_vec.doImport(serial_req_vec,importOperatorElem,Tpetra::INSERT);

        // Extracting the mesh field and the tpetra vector view
        QPScalarFieldType* field = metaData->get_field<QPScalarFieldType>(stk::topology::ELEM_RANK, *it);
        TEUCHOS_TEST_FOR_EXCEPTION (field==0, std::logic_error, "Error! Field " << *it << " not present (perhaps is 'Node Scalar'?).\n");

        Teuchos::ArrayRCP<const ST> req_vec_view = req_vec.get1dView();

        //Now we have to stuff the vector in the mesh data
        for (int i(0); i<elems.size(); ++i)
        {
          elemId = bulkData->identifier(elems[i]) - 1;
          lid    = elems_map->getLocalElement((GO)(elemId));

          values = stk::mesh::field_data(*field, elems[i]);
          values[0] = req_vec_view[lid];
        }
      }
      else if (ftype == "Node Vector")
      {
        // Loading the dimension of the Vector Field (by default equal to the mesh dimension)
        temp_str = *it + " Field Dimension";
        int fieldDim = req_fields_info->get<int>(temp_str,this->meshSpecs[0]->numDim);

        // Creating the serial and (possibly) parallel Tpetra service multivectors
        Tpetra_MultiVector serial_req_mvec(serial_nodes_map,fieldDim);
        Tpetra_MultiVector req_mvec(nodes_map,fieldDim);

        temp_str = *it + " Value";
        if (req_fields_info->isParameter(temp_str))
        {
          *out << "Discarding other info about Node Vector field " << *it << " and filling it with constant value "
               << req_fields_info->get<Teuchos::Array<double> >(temp_str) << "\n";
          // For debug, we allow to fill the field with a given uniform value
          fillTpetraMVec (serial_req_mvec,req_fields_info->get<Teuchos::Array<double> >(temp_str));
        }
        else if (fname!="")
        {
          *out << "Reading Node Vector field " << *it << " from file " << fname << "\n";
          // Read the input file and stuff it in the Tpetra multivector
          readVectorFileSerial (fname,serial_req_mvec,commT);

          temp_str = *it + " Scale Factor";
          if (req_fields_info->isParameter(temp_str))
          {
            double scale_factor = req_fields_info->get<double>(temp_str);
            serial_req_mvec.scale (scale_factor);
          }
        }
        else
        {
          bool found = false;
          for (int i(0); i<missing.size(); ++i)
          {
            if (missing[i].field()->name()==*it)
            {
              *out << "No file name nor constant value specified for Node Vector field " << *it << "; initializing it to 0.\n";
              Teuchos::Array<double> vals(fieldDim,0.);
              fillTpetraMVec (serial_req_mvec,vals);
              found = true;
              break;
            }
          }
          if (!found)
          {
            *out << "Using mesh-stored values for Node Vector field " << *it << " since no constant value nor filename has been specified\n";
            continue;
          }
        }

        // Fill the (possibly) parallel vector
        req_mvec.doImport(serial_req_mvec,importOperatorNode,Tpetra::INSERT);

        // Extracting the mesh field and the tpetra vector views
        VectorFieldType* field = metaData->get_field<VectorFieldType>(stk::topology::NODE_RANK, *it);
        TEUCHOS_TEST_FOR_EXCEPTION (field==0, std::logic_error, "Error! Field " << *it << " not present (perhaps is 'Elem Vector'?).\n");

        std::vector<Teuchos::ArrayRCP<const ST> > req_mvec_view;
        for (int i(0); i<fieldDim; ++i)
          req_mvec_view.push_back(req_mvec.getVector(i)->get1dView());

        //Now we have to stuff the vector in the mesh data
        for (int i(0); i<nodes.size(); ++i)
        {
          nodeId = bulkData->identifier(nodes[i]) - 1;
          lid    = nodes_map->getLocalElement((GO)(nodeId));

          values = stk::mesh::field_data(*field, nodes[i]);

          for (int iDim(0); iDim<fieldDim; ++iDim)
            values[iDim] = req_mvec_view[iDim][lid];
        }
      }
      else if (ftype == "Elem Vector")
      {
        // Loading the dimension of the Vector Field (by default equal to the mesh dimension)
        temp_str = *it + " Field Dimension";
        int fieldDim = req_fields_info->get<int>(temp_str,this->meshSpecs[0]->numDim);

        // Creating the serial and (possibly) parallel Tpetra service multivectors
        Tpetra_MultiVector serial_req_mvec(serial_elems_map,fieldDim);
        Tpetra_MultiVector req_mvec(elems_map,fieldDim);

        temp_str = *it + " Value";
        if (req_fields_info->isParameter(temp_str))
        {
          *out << "Discarding other info about Elem Vector field " << *it << " and filling it with constant value "
               << req_fields_info->get<Teuchos::Array<double> >(temp_str) << "\n";
          // For debug, we allow to fill the field with a given uniform value
          fillTpetraMVec (serial_req_mvec,req_fields_info->get<Teuchos::Array<double> >(temp_str));
        }
        else if (fname!="")
        {
          *out << "Reading Elem Vector field " << *it << " from file " << fname << "\n";
          // Read the input file and stuff it in the Tpetra multivector
          readVectorFileSerial (fname,serial_req_mvec,commT);

          temp_str = *it + " Scale Factor";
          if (req_fields_info->isParameter(temp_str))
          {
            double scale_factor = req_fields_info->get<double>(temp_str);
            serial_req_mvec.scale (scale_factor);
          }
        }
        else
        {
          bool found = false;
          for (int i(0); i<missing.size(); ++i)
          {
            if (missing[i].field()->name()==*it)
            {
              *out << "No file name nor constant value specified for Elem Vector field " << *it << "; initializing it to 0.\n";
              Teuchos::Array<double> vals(fieldDim,0.);
              fillTpetraMVec (serial_req_mvec,vals);
              found = true;
              break;
            }
          }
          if (!found)
          {
            *out << "Using mesh-stored values for Elem Vector field " << *it << " since no constant value nor filename has been specified\n";
            continue;
          }
        }

        // Fill the (possibly) parallel vector
        req_mvec.doImport(serial_req_mvec,importOperatorElem,Tpetra::INSERT);

        // Extracting the mesh field and the tpetra vector views
        VectorFieldType* field = metaData->get_field<VectorFieldType>(stk::topology::ELEM_RANK, *it);
        TEUCHOS_TEST_FOR_EXCEPTION (field==0, std::logic_error, "Error! Field " << *it << " not present (perhaps is 'Node Vector'?).\n");
        std::vector<Teuchos::ArrayRCP<const ST> > req_mvec_view;
        for (int i(0); i<fieldDim; ++i)
          req_mvec_view.push_back(req_mvec.getVector(i)->get1dView());

        //Now we have to stuff the vector in the mesh data
        for (int i(0); i<elems.size(); ++i)
        {
          elemId = bulkData->identifier(elems[i]) - 1;
          lid    = elems_map->getLocalElement((GO)(elemId));

          values = stk::mesh::field_data(*field, nodes[i]);

          for (int iDim(0); iDim<fieldDim; ++iDim)
            values[iDim] = req_mvec_view[iDim][lid];
        }
      }
      else
      {
        TEUCHOS_TEST_FOR_EXCEPTION (true, Teuchos::Exceptions::InvalidParameterValue,
                                    "Sorry, I haven't yet implemented the case of field that are not Scalar nor Vector or that is not at nodal nor elemental.\n");
      }
    }
  }

  if (params->get<bool>("Write points coordinates to ascii file", false))
  {
    AbstractSTKFieldContainer::VectorFieldType* coordinates_field = fieldContainer->getCoordinatesField();
    std::ofstream ofile;
    ofile.open("coordinates.ascii");
    if (!ofile.is_open())
    {
      TEUCHOS_TEST_FOR_EXCEPTION (true, std::logic_error, "Error! Cannot open coordinates file.\n");
    }

    ofile << nodes.size() << " " << 4 << "\n";

    stk::mesh::Entity node;

    for (int i(0); i<nodes.size(); ++i)
    {
      node   = bulkData->get_entity(stk::topology::NODE_RANK, i + 1);
      double* coord = stk::mesh::field_data(*coordinates_field, node);

      ofile << bulkData->identifier (nodes[i]) << " " << coord[0] << " " << coord[1]  << " " << coord[2] << "\n";
    }

    ofile.close();
  }
#endif

  // Refine the mesh before starting the simulation if indicated
  uniformRefineMesh(commT);

  // Rebalance the mesh before starting the simulation if indicated
  rebalanceInitialMeshT(commT);

  // Build additional mesh connectivity needed for mesh fracture (if indicated)
  computeAddlConnectivity();
}
Albany::OrdinarySTKFieldContainer<Interleaved>::OrdinarySTKFieldContainer(
  const Teuchos::RCP<Teuchos::ParameterList>& params_,
  stk::mesh::fem::FEMMetaData* metaData_,
  stk::mesh::BulkData* bulkData_,
  const int neq_,
  const AbstractFieldContainer::FieldContainerRequirements& req,
  const int numDim_,
  const Teuchos::RCP<Albany::StateInfoStruct>& sis)
  : GenericSTKFieldContainer<Interleaved>(params_, metaData_, bulkData_, neq_, numDim_),
      buildSurfaceHeight(false),
      buildTemperature(false),
      buildBasalFriction(false),
      buildThickness(false),
      buildFlowFactor(false),
      buildSurfaceVelocity(false),
      buildVelocityRMS(false) {

  typedef typename AbstractSTKFieldContainer::VectorFieldType VFT;
  typedef typename AbstractSTKFieldContainer::ScalarFieldType SFT;

#ifdef ALBANY_FELIX
  buildSurfaceHeight = (std::find(req.begin(), req.end(), "Surface Height") != req.end());

  buildTemperature =  (std::find(req.begin(), req.end(), "Temperature") != req.end());

  buildBasalFriction = (std::find(req.begin(), req.end(), "Basal Friction") != req.end());

  buildThickness = (std::find(req.begin(), req.end(), "Thickness") != req.end());
  
  buildFlowFactor =  (std::find(req.begin(), req.end(), "Flow Factor") != req.end());

  buildSurfaceVelocity = (std::find(req.begin(), req.end(), "Surface Velocity") != req.end());

  buildVelocityRMS = (std::find(req.begin(), req.end(), "Velocity RMS") != req.end());
#endif

  //Start STK stuff
  this->coordinates_field = & metaData_->declare_field< VFT >("coordinates");
  solution_field = & metaData_->declare_field< VFT >(
                     params_->get<std::string>("Exodus Solution Name", "solution"));

#ifdef ALBANY_LCM
  residual_field = & metaData_->declare_field< VFT >(
                     params_->get<std::string>("Exodus Residual Name", "residual"));
#endif

#ifdef ALBANY_FELIX

  if(buildSurfaceHeight)
    this->surfaceHeight_field = & metaData_->declare_field< SFT >("surface_height");
  if(buildTemperature)
    this->temperature_field = & metaData_->declare_field< SFT >("temperature");
  if(buildBasalFriction)
	  this->basalFriction_field = & metaData_->declare_field< SFT >("basal_friction");
  if(buildThickness)
    this->thickness_field = & metaData_->declare_field< SFT >("thickness");
  if(buildFlowFactor)
    this->flowFactor_field = & metaData_->declare_field< SFT >("flow_factor");
  if(buildSurfaceVelocity)
    this->surfaceVelocity_field = & metaData_->declare_field< VFT >("surface_velocity");
  if(buildVelocityRMS)
    this->velocityRMS_field = & metaData_->declare_field< VFT >("velocity_RMS");
#endif

  stk::mesh::put_field(*this->coordinates_field , metaData_->node_rank() , metaData_->universal_part(), numDim_);
  stk::mesh::put_field(*solution_field , metaData_->node_rank() , metaData_->universal_part(), neq_);

#ifdef ALBANY_LCM
  stk::mesh::put_field(*residual_field , metaData_->node_rank() , metaData_->universal_part() , neq_);
#endif

#ifdef ALBANY_FELIX

  if(buildSurfaceHeight)
    stk::mesh::put_field( *this->surfaceHeight_field , metaData_->node_rank() , metaData_->universal_part());
  if(buildTemperature)
    stk::mesh::put_field( *this->temperature_field , metaData_->element_rank() , metaData_->universal_part());
  if(buildBasalFriction)
    stk::mesh::put_field( *this->basalFriction_field , metaData_->node_rank() , metaData_->universal_part());//*metaData_->get_part("basalside","Mpas Interface"));
  if(buildThickness)
    stk::mesh::put_field( *this->thickness_field , metaData_->node_rank() , metaData_->universal_part());
  if(buildFlowFactor)
    stk::mesh::put_field( *this->flowFactor_field , metaData_->element_rank() , metaData_->universal_part());
  if(buildSurfaceVelocity)
    stk::mesh::put_field( *this->surfaceVelocity_field , metaData_->node_rank() , metaData_->universal_part(), neq_);
  if(buildVelocityRMS)
    stk::mesh::put_field( *this->velocityRMS_field , metaData_->node_rank() , metaData_->universal_part(), neq_);
#endif

#ifdef ALBANY_SEACAS
  stk::io::set_field_role(*this->coordinates_field, Ioss::Field::MESH);
  stk::io::set_field_role(*solution_field, Ioss::Field::TRANSIENT);
#ifdef ALBANY_LCM
  stk::io::set_field_role(*residual_field, Ioss::Field::TRANSIENT);
#endif

#ifdef ALBANY_FELIX

  // ATTRIBUTE writes only once per file, but somehow did not work on restart.
  //stk::io::set_field_role(*surfaceHeight_field, Ioss::Field::ATTRIBUTE);
  if(buildSurfaceHeight)
     stk::io::set_field_role(*this->surfaceHeight_field, Ioss::Field::TRANSIENT);
  if(buildTemperature)
     stk::io::set_field_role(*this->temperature_field, Ioss::Field::TRANSIENT);
  if(buildBasalFriction)
     stk::io::set_field_role(*this->basalFriction_field, Ioss::Field::TRANSIENT);
  if(buildThickness)
     stk::io::set_field_role(*this->thickness_field, Ioss::Field::TRANSIENT);
  if(buildFlowFactor)
     stk::io::set_field_role(*this->flowFactor_field, Ioss::Field::TRANSIENT);
  if(buildSurfaceVelocity)
     stk::io::set_field_role(*this->surfaceVelocity_field, Ioss::Field::TRANSIENT);
  if(buildVelocityRMS)
     stk::io::set_field_role(*this->velocityRMS_field, Ioss::Field::TRANSIENT);
#endif
#endif


  // If the problem requests that the initial guess at the solution equals the input node coordinates,
  // set that here
  /*
    if(std::find(req.begin(), req.end(), "Initial Guess Coords") != req.end()){
       this->copySTKField(this->coordinates_field, solution_field);
    }
  */



  this->buildStateStructs(sis);

  initializeSTKAdaptation();

}
Albany::MultiSTKFieldContainer<Interleaved>::MultiSTKFieldContainer(
  const Teuchos::RCP<Teuchos::ParameterList>& params_,
  const Teuchos::RCP<stk::mesh::MetaData>& metaData_,
  const Teuchos::RCP<stk::mesh::BulkData>& bulkData_,
  const int neq_,
  const AbstractFieldContainer::FieldContainerRequirements& req,
  const int numDim_,
  const Teuchos::RCP<Albany::StateInfoStruct>& sis,
  const Teuchos::Array<Teuchos::Array<std::string> >& solution_vector,
  const Teuchos::Array<std::string>& residual_vector)
  : GenericSTKFieldContainer<Interleaved>(params_, metaData_, bulkData_, neq_, numDim_),
    haveResidual(false), buildSphereVolume(false) {

  typedef typename AbstractSTKFieldContainer::VectorFieldType VFT;
  typedef typename AbstractSTKFieldContainer::ScalarFieldType SFT;
  typedef typename AbstractSTKFieldContainer::SphereVolumeFieldType SVFT;

  sol_vector_name.resize(solution_vector.size());
  sol_index.resize(solution_vector.size());

  // Check the input

  for(int vec_num = 0; vec_num < solution_vector.size(); vec_num++){

    if(solution_vector[vec_num].size() == 0) { // Do the default solution vector

      std::string name = params_->get<std::string>(sol_tag_name[vec_num], sol_id_name[vec_num]);
      VFT* solution = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, name);
      stk::mesh::put_field(*solution, metaData_->universal_part(), neq_);
#ifdef ALBANY_SEACAS
      stk::io::set_field_role(*solution, Ioss::Field::TRANSIENT);
#endif

      sol_vector_name[vec_num].push_back(name);
      sol_index[vec_num].push_back(this->neq);

    }

    else if(solution_vector[vec_num].size() == 1) { // User is just renaming the entire solution vector

      VFT* solution = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, solution_vector[vec_num][0]);
      stk::mesh::put_field(*solution, metaData_->universal_part(), neq_);
#ifdef ALBANY_SEACAS
      stk::io::set_field_role(*solution, Ioss::Field::TRANSIENT);
#endif

      sol_vector_name[vec_num].push_back(solution_vector[vec_num][0]);
      sol_index[vec_num].push_back(neq_);

    }

    else { // user is breaking up the solution into multiple fields

      // make sure the number of entries is even

      TEUCHOS_TEST_FOR_EXCEPTION((solution_vector[vec_num].size() % 2), std::logic_error,
                               "Error in input file: specification of solution vector layout is incorrect." << std::endl);

      int len, accum = 0;

      for(int i = 0; i < solution_vector[vec_num].size(); i += 2) {

        if(solution_vector[vec_num][i + 1] == "V") {

          len = numDim_; // vector
          accum += len;
          VFT* solution = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, solution_vector[vec_num][i]);
          stk::mesh::put_field(*solution, metaData_->universal_part(), len);
#ifdef ALBANY_SEACAS
          stk::io::set_field_role(*solution, Ioss::Field::TRANSIENT);
#endif
          sol_vector_name[vec_num].push_back(solution_vector[vec_num][i]);
          sol_index[vec_num].push_back(len);

        }

        else if(solution_vector[vec_num][i + 1] == "S") {

          len = 1; // scalar
          accum += len;
          SFT* solution = & metaData_->declare_field< SFT >(stk::topology::NODE_RANK, solution_vector[vec_num][i]);
          stk::mesh::put_field(*solution, metaData_->universal_part());
#ifdef ALBANY_SEACAS
          stk::io::set_field_role(*solution, Ioss::Field::TRANSIENT);
#endif
          sol_vector_name[vec_num].push_back(solution_vector[vec_num][i]);
          sol_index[vec_num].push_back(len);

        }

        else

          TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
                                   "Error in input file: specification of solution vector layout is incorrect." << std::endl);

      }

      TEUCHOS_TEST_FOR_EXCEPTION(accum != neq_, std::logic_error,
                               "Error in input file: specification of solution vector layout is incorrect." << std::endl);

    }
  }

#if defined(ALBANY_LCM)
  // do the residual next

  if(residual_vector.size() == 0) { // Do the default residual vector

    std::string name = params_->get<std::string>(res_tag_name, res_id_name);
    VFT* residual = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, name);
    stk::mesh::put_field(*residual, metaData_->universal_part(), neq_);
#ifdef ALBANY_SEACAS
    stk::io::set_field_role(*residual, Ioss::Field::TRANSIENT);
#endif

    res_vector_name.push_back(name);
    res_index.push_back(neq_);

  }

  else if(residual_vector.size() == 1) { // User is just renaming the entire residual vector

    VFT* residual = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, residual_vector[0]);
    stk::mesh::put_field(*residual, metaData_->universal_part(), neq_);
#ifdef ALBANY_SEACAS
    stk::io::set_field_role(*residual, Ioss::Field::TRANSIENT);
#endif

    res_vector_name.push_back(residual_vector[0]);
    res_index.push_back(neq_);

  }

  else { // user is breaking up the residual into multiple fields

    // make sure the number of entries is even

    TEUCHOS_TEST_FOR_EXCEPTION((residual_vector.size() % 2), std::logic_error,
                               "Error in input file: specification of residual vector layout is incorrect." << std::endl);

    int len, accum = 0;

    for(int i = 0; i < residual_vector.size(); i += 2) {

      if(residual_vector[i + 1] == "V") {

        len = numDim_; // vector
        accum += len;
        VFT* residual = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, residual_vector[i]);
        stk::mesh::put_field(*residual, metaData_->universal_part(), len);
#ifdef ALBANY_SEACAS
        stk::io::set_field_role(*residual, Ioss::Field::TRANSIENT);
#endif
        res_vector_name.push_back(residual_vector[i]);
        res_index.push_back(len);

      }

      else if(residual_vector[i + 1] == "S") {

        len = 1; // scalar
        accum += len;
        SFT* residual = & metaData_->declare_field< SFT >(stk::topology::NODE_RANK, residual_vector[i]);
        stk::mesh::put_field(*residual, metaData_->universal_part());
#ifdef ALBANY_SEACAS
        stk::io::set_field_role(*residual, Ioss::Field::TRANSIENT);
#endif
        res_vector_name.push_back(residual_vector[i]);
        res_index.push_back(len);

      }

      else

        TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
                                   "Error in input file: specification of residual vector layout is incorrect." << std::endl);

    }

    TEUCHOS_TEST_FOR_EXCEPTION(accum != neq_, std::logic_error,
                               "Error in input file: specification of residual vector layout is incorrect." << std::endl);

  }

  haveResidual = true;

#endif

  //Do the coordinates
  this->coordinates_field = & metaData_->declare_field< VFT >(stk::topology::NODE_RANK, "coordinates");
  stk::mesh::put_field(*this->coordinates_field , metaData_->universal_part(), numDim_);
#ifdef ALBANY_SEACAS
  stk::io::set_field_role(*this->coordinates_field, Ioss::Field::MESH);
#endif

#if defined(ALBANY_LCM) && defined(ALBANY_SEACAS)
  // sphere volume is a mesh attribute read from a genesis mesh file containing sphere element (used for peridynamics)
  bool hasSphereVolumeFieldContainerRequirement = (std::find(req.begin(), req.end(), "Sphere Volume") != req.end());
  if(hasSphereVolumeFieldContainerRequirement){
    this->sphereVolume_field = metaData_->template get_field< SVFT >(stk::topology::ELEMENT_RANK, "volume");
    if(this->sphereVolume_field != 0){
      buildSphereVolume = true;
      stk::io::set_field_role(*this->sphereVolume_field, Ioss::Field::ATTRIBUTE);
    }
  }
#endif

  this->addStateStructs(sis);

  initializeSTKAdaptation();

}