//======================================================================================================================
//======================================================================================================================
//======================================================================================================================
STKUNIT_UNIT_TEST(perceptMesh, open_new_close_PerceptMesh_3)
{
    EXCEPTWATCH;

    // start_demo_open_new_close_PerceptMesh_3
    PerceptMesh eMesh(3u);
    eMesh.new_mesh(GMeshSpec("3x3x3|bbox:0,0,0,1,1,1"));  // create a 3x3x3 hex mesh in the unit cube
    int scalarDimension = 0; // a scalar
    int vectorDimension = 3;

    mesh::FieldBase* pressure_field = eMesh.add_field("pressure", stk::mesh::fem::FEMMetaData::NODE_RANK, scalarDimension);
    eMesh.add_field("velocity", stk::mesh::fem::FEMMetaData::NODE_RANK, vectorDimension);
    eMesh.add_field("element_volume", eMesh.element_rank(), scalarDimension);

    eMesh.commit();

    EXPECT_CATCH( eMesh.commit() , commit_again);

    // create a field function from the new pressure field
    FieldFunction ff_pressure("ff_pressure", pressure_field, eMesh, 3, 1);

    // set the value of the pressure field to a constant everywhere
    ConstantFunction initPressureValues(pressure_value, "initPVal");
    ff_pressure.interpolateFrom(initPressureValues);

    // save
    eMesh.save_as(output_files_loc+"cube_with_pressure_3.e");
    eMesh.close();

    EXPECT_CATCH( eMesh.print_info("bad", 1) , mesh_closed_try_print);

    // end_demo

}
STKUNIT_UNIT_TEST(geom, volume)
{
  dw().m(LOG_GEOMETRY_VERIFIER) << "TEST::geom::volume " << stk_classic::diag::dendl;

  const size_t num_x = 3;
  const size_t num_y = 3;
  const size_t num_z = 3;
  std::string config_mesh =
    Ioss::Utils::to_string(num_x) + "x" +
    Ioss::Utils::to_string(num_y) + "x" +
    Ioss::Utils::to_string(num_z) + "|bbox:0,0,0,1,1,1";
	
  PerceptMesh eMesh(3u);
  eMesh.new_mesh(GMeshSpec(config_mesh));
  eMesh.commit();
  // no need for this in create mode: eMesh.readBulkData();

  //FEMMetaData& metaData = *eMesh.get_fem_meta_data();
  mesh::BulkData& bulkData = *eMesh.get_bulk_data();

  eMesh.dump();
  GeometryVerifier geomVerifier(false);
  geomVerifier.isGeometryBad(bulkData, true);
  //setDoPause(true);
  //pause();
}
//======================================================================================================================
//======================================================================================================================
//======================================================================================================================
STKUNIT_UNIT_TEST(perceptMesh, open_new_close_PerceptMesh_2)
{
    EXCEPTWATCH;

    double x=0.123, y=0.234, z=0.345, time=0.0;

    // start_demo_open_new_close_PerceptMesh_2
    PerceptMesh eMesh(3u);
    // open the file we previously saved with the new fields
    eMesh.open_read_only(input_files_loc+"cube_with_pressure.e");

    eMesh.print_info("Info after reading mesh");

    //eMesh.print_fields();
    mesh::FieldBase *f_coords = eMesh.get_field("coordinates");

    // create a field function from the existing coordinates field
    FieldFunction ff_coords("ff_coords", f_coords, eMesh, 3, 3);

    // here we could evaluate this field function
    eval_vec3_print(x, y, z, time, ff_coords);

    // get the pressure field
    mesh::FieldBase* pressure_field = eMesh.get_field("pressure");

    // FIXME
    std::vector< const mesh::FieldBase * > sync_fields( 1 , pressure_field );
    mesh::communicate_field_data( eMesh.get_bulk_data()->shared_aura() , sync_fields );
    // FIXME

    //double * pdata = eMesh.node_field_data(pressure_field, 1);

    FieldFunction ff_pressure("ff_pressure", pressure_field, eMesh, 3, 1);
    ff_pressure.add_alias("P");
    StringFunction sf_pressure("P");

    // a point-source at the origin
#define EXACT_SOL log(sqrt( x*x + y*y + z*z) + 1.e-10)

    StringFunction sf_exact_solution(EXPAND_AND_QUOTE(EXACT_SOL), Name("sf_exact_solution"), 3, 1);
    StringFunction sf_error = sf_exact_solution - sf_pressure;

    eval_print(x,y,z,time, sf_error);
    double val_cpp = EXACT_SOL - pressure_value;
    double val_sf  = eval(x,y,z,time, sf_error);
    EXPECT_DOUBLE_EQ(val_sf, val_cpp);
    // end_demo

}
//======================================================================================================================
//======================================================================================================================
//======================================================================================================================
STKUNIT_UNIT_TEST(perceptMesh, open_new_reopen_PerceptMesh)
{
    EXCEPTWATCH;

    // start_demo_open_new_reopen_PerceptMesh
    PerceptMesh eMesh(3u);
    eMesh.new_mesh(GMeshSpec("3x3x3|bbox:0,0,0,1,1,1"));  // create a 3x3x3 hex mesh in the unit cube
    int scalarDimension = 0; // a scalar
    int vectorDimension = 3;

    eMesh.add_field("pressure", stk::mesh::fem::FEMMetaData::NODE_RANK, scalarDimension);
    eMesh.add_field("velocity", stk::mesh::fem::FEMMetaData::NODE_RANK, vectorDimension);
    eMesh.add_field("element_volume", eMesh.element_rank(), scalarDimension);

    eMesh.commit();

    /// reopen the mesh to allow for more fields to be added - note that this involves a db write/read operation
    eMesh.reopen(output_files_loc+"optional_temp_filename.e");
    mesh::FieldBase* momentum_field = eMesh.add_field("momentum", stk::mesh::fem::FEMMetaData::NODE_RANK, vectorDimension);
    eMesh.commit();

    // create a field function from the new pressure field
    mesh::FieldBase *pressure_field = eMesh.get_field("pressure");
    FieldFunction ff_pressure("ff_pressure", pressure_field, eMesh, 3, 1);

    // set the value of the pressure field to a constant everywhere
    ConstantFunction initPressureValues(pressure_value, "initPVal");
    ff_pressure.interpolateFrom(initPressureValues);

    // set the momentum field
    std::vector<double> momentum_values(3);
    momentum_values[0] = 2034.5;
    momentum_values[1] = 2134.5;
    momentum_values[2] = 2234.5;
    ConstantFunctionVec initMomentumValues(momentum_values, "initMomVal");

    // create a field function from the new momentum field
    FieldFunction ff_momentum("ff_momentum", momentum_field, eMesh, 3, 3);
    ff_momentum.interpolateFrom(initMomentumValues);

    // save
    eMesh.save_as(output_files_loc+"cube_with_pressure_and_momentum.e");
    eMesh.close();


    // end_demo

}
STKUNIT_UNIT_TEST(adapt, count_memory)
{
  stk_classic::ParallelMachine pm = MPI_COMM_WORLD ;

  const unsigned p_size = stk_classic::parallel_machine_size( pm );
  if (p_size == 1)
    {
      const unsigned n = 20;
      //const unsigned nx = n , ny = n , nz = p_size*n ;
      const unsigned nx = n , ny = n;

      percept::QuadFixture<double, shards::Triangle<3> > fixture( pm , nx , ny, false);
      fixture.meta_data.commit();
      fixture.generate_mesh();

      percept::PerceptMesh eMesh(&fixture.meta_data, &fixture.bulk_data);
      //eMesh.print_info("quad mesh",2);

      //const size_t num_new_tris = 2000*2000;
      const size_t num_new_tris = 20*20;
      const size_t num_nodes_per_tri = 3;
      const size_t num_new_nodes = num_new_tris*num_nodes_per_tri;
      MemoryInfo mem_delta_node;
      double time = -stk_classic::percept::Util::cpu_time();

      std::vector<stk_classic::mesh::Entity *> new_nodes, new_elements;

      eMesh.get_bulk_data()->modification_begin();
      eMesh.createEntities(eMesh.node_rank(), num_new_nodes, new_nodes);
      eMesh.get_bulk_data()->modification_end();

      mem_delta_node.get_increment();
      double mem_per_node = double(mem_delta_node.m_malloc_used)/double(num_new_nodes);
      std::cout << "\nstk_mesh count_memory mem_per_node = " << mem_per_node << "\n" << std::endl;

      MemoryInfo mem_delta_elem_0, mem_delta_elem_1;

      eMesh.get_bulk_data()->modification_begin();
      eMesh.createEntities(eMesh.element_rank(), num_new_tris, new_elements);
      eMesh.get_bulk_data()->modification_end();

      mem_delta_elem_0.get_increment();

      eMesh.get_bulk_data()->modification_begin();
      size_t i_node=0;
      for (size_t i=0; i<num_new_tris; ++i) {
        unsigned ordinal = 0;
        for (size_t j=0; j < num_nodes_per_tri; ++j) {
          eMesh.get_bulk_data()->declare_relation(*new_elements[i],*new_nodes[i_node],ordinal);
          ++ordinal;
          ++i_node;
        }
      }
      eMesh.get_bulk_data()->modification_end();

      mem_delta_elem_1.get_increment();
      double mem_per_elem_0 = double(mem_delta_elem_0.m_malloc_used)/double(num_new_tris);
      double mem_per_elem_1 = double(mem_delta_elem_1.m_malloc_used)/double(num_new_tris);

      time += stk_classic::percept::Util::cpu_time();

      std::cout << "\nstk_mesh count_memory mem_per_elem (no connectivity) = " << mem_per_elem_0 << " with connectivity= " << mem_per_elem_1 << " cpu= " << time << std::endl;

    }
}
//======================================================================================================================
//======================================================================================================================
//======================================================================================================================
STKUNIT_UNIT_TEST(perceptMesh, open_new_close_PerceptMesh)
{
    EXCEPTWATCH;

    // start_demo_open_new_close_PerceptMesh
    PerceptMesh eMesh(3u);
    eMesh.new_mesh(GMeshSpec("3x3x3|bbox:0,0,0,1,1,1"));  // create a 3x3x3 hex mesh in the unit cube
    int scalarDimension = 0; // a scalar
    int vectorDimension = 3;

    mesh::FieldBase* pressure_field = eMesh.add_field("pressure", stk::mesh::fem::FEMMetaData::NODE_RANK, scalarDimension);
    eMesh.add_field("velocity", stk::mesh::fem::FEMMetaData::NODE_RANK, vectorDimension);
    eMesh.add_field("element_volume", eMesh.element_rank(), scalarDimension);

    eMesh.commit();

    // create a field function from the new pressure field
    FieldFunction ff_pressure("ff_pressure", pressure_field, eMesh, 3, 1);

    // set the value of the pressure field to a constant everywhere
    ConstantFunction initPressureValues(pressure_value, "initPVal");
    ff_pressure.interpolateFrom(initPressureValues);

    //if (eMesh.get_rank()== 0) eMesh.print_fields("Pressure");
    //exit(1);

    // here we could evaluate this field function
    double x=0.123, y=0.234, z=0.345, time=0.0;
    std::cout << "P[" << eMesh.get_rank() << "] "
              << "before write ff_pressure = " << eval(x,y,z,time, ff_pressure) << std::endl;

    //eval_print(x, y, z, time, ff_pressure);

    double pval = eval(x, y, z, time, ff_pressure);
    EXPECT_DOUBLE_EQ(pval, pressure_value);

    eMesh.save_as(output_files_loc+"cube_with_pressure.e");
    eMesh.close();

    // end_demo

    // start_demo_open_new_close_PerceptMesh_1
    // open the file we previously saved with the new fields
    eMesh.open_read_only(input_files_loc+"cube_with_pressure.e");

    // get the pressure field
    pressure_field = eMesh.get_field("pressure");

    // FIXME
    std::vector< const mesh::FieldBase * > sync_fields( 1 , pressure_field );
    mesh::communicate_field_data( eMesh.get_bulk_data()->shared_aura() , sync_fields );
    // FIXME

    //if (1 || eMesh.get_rank()== 0) eMesh.print_fields("Pressure");

    FieldFunction ff_pressure_1("ff_pressure", pressure_field, eMesh, 3, 1);
    ff_pressure_1.add_alias("P");
    StringFunction sf_pressure("P");
    std::cout << "P[" << eMesh.get_rank() << "] "
              << "after read ff_pressure = " << eval(x,y,z,time, ff_pressure_1) << std::endl;

    // a point-source at the origin
#define EXACT_SOL log(sqrt( x*x + y*y + z*z) + 1.e-10)

    StringFunction sf_exact_solution(EXPAND_AND_QUOTE(EXACT_SOL), Name("sf_exact_solution"), 3, 1);
    StringFunction sf_error = sf_exact_solution - sf_pressure;

    std::cout << "P[" << eMesh.get_rank() << "] "
              << "sf_pressure = " << eval(x,y,z,time, sf_pressure) << std::endl;
    //!eval_print(x,y,z,time, sf_error);
    std::cout << "P[" << eMesh.get_rank() << "] "
              << "sf_error = " << eval(x,y,z,time, sf_error) << std::endl;
    double val_cpp = EXACT_SOL - pressure_value;
    double val_sf  = eval(x,y,z,time, sf_error);
    EXPECT_DOUBLE_EQ(val_sf, val_cpp);
    // end_demo

}
/// This test uses a back door to the function that passes in the element to avoid the lookup of the element when the
///  StringFunction contains references to FieldFunctions
void TEST_norm_string_function_turbo_timings(TurboOption turboOpt)
{
  EXCEPTWATCH;
  //stk::diag::WriterThrowSafe _write_throw_safe(dw());
  //dw().setPrintMask(dw_option_mask.parse(vm["dw"].as<std::string>().c_str()));
  //dw().setPrintMask(LOG_NORM+LOG_ALWAYS);

  dw().m(LOG_NORM) << "TEST.norm.string_function " << stk::diag::dendl;

  /// create a meta data/bulk data empty pair
  PerceptMesh eMesh(3u);

  if (1)
  {
    // Need a symmetric mesh around the origin for some of the tests below to work correctly (i.e. have analytic solutions)
    const size_t nxyz = 4;
    const size_t num_x = nxyz;
    const size_t num_y = nxyz;
    const size_t num_z = nxyz;
    std::string config_mesh =
      Ioss::Utils::to_string(num_x) + "x" +
      Ioss::Utils::to_string(num_y) + "x" +
      Ioss::Utils::to_string(num_z) + "|bbox:-0.5,-0.5,-0.5,0.5,0.5,0.5";
	
    eMesh.new_mesh(GMeshSpec(config_mesh));

    eMesh.commit();
  }

  mesh::fem::FEMMetaData& metaData = *eMesh.get_fem_meta_data();
  mesh::BulkData& bulkData = *eMesh.get_bulk_data();

  /// the coordinates field is always created by the PerceptMesh read operation, here we just get the field
  mesh::FieldBase *coords_field = metaData.get_field<mesh::FieldBase>("coordinates");

  /// create a field function from the existing coordinates field
  FieldFunction ff_coords("ff_coords", coords_field, &bulkData,
                          Dimensions(3), Dimensions(3), FieldFunction::SIMPLE_SEARCH );

  /// the function to be integrated:  sqrt(Integral[x^2, dxdydz]) =?= sqrt(x^3/3 @ [-0.5, 0.5]) ==> sqrt(0.25/3)
  StringFunction sfx("x", Name("sfx"), Dimensions(3), Dimensions(1) );

  ff_coords.add_alias("mc");
  //StringFunction sfcm("sqrt(mc[0]*mc[0]+mc[1]*mc[1]+mc[2]*mc[2])", Name("sfcm"), Dimensions(3), Dimensions(1));
  StringFunction sfx_mc("mc[0]", Name("sfx_mc"), Dimensions(3), Dimensions(1) );

  /// the function to be integrated:  sqrt(Integral[x^2, dxdydz]) =?= sqrt(x^3/3 @ [-0.5, 0.5]) ==> sqrt(0.25/3)

  /// A place to hold the result.
  /// This is a "writable" function (we may want to make this explicit - StringFunctions are not writable; FieldFunctions are
  /// since we interpolate values to them from other functions).
  ConstantFunction sfx_res(0.0, "sfx_res");
  ConstantFunction sfx_res_turbo(0.0, "sfx_res_turbo");
  ConstantFunction sfx_res_slow(0.0, "sfx_res_slow");
  ConstantFunction sfx_res_fast(0.0, "sfx_res_fast");

#define COL_SEP "|"
#define EXPR_CELL_WIDTH (80)

#define TIME_IT2(expr_none,expr_turbo,msg,topt)                         \
  {                                                                     \
    double TURBO_NONE_time    = 0;                                      \
    double TURBO_ON_time = 0;                                           \
    TIME_IT(expr_none,TURBO_NONE_time);                                 \
    TIME_IT(expr_turbo,TURBO_ON_time);                                  \
    if (1) std::cout << msg << #topt << "for expression= " << QUOTE(expr_none) << " timings= " << std::endl; \
    if (1) std::cout << "TURBO_NONE_time= " << TURBO_NONE_time << " "   \
                     << ( turboOpt==TURBO_ELEMENT?"TURBO_ELEMENT_time":"TURBO_BUCKET_time") <<"= " << TURBO_ON_time \
                     << " ratio= " << TURBO_NONE_time/TURBO_ON_time << std::endl; \
  }

  int numIter = 1;
  for (int iter = 0; iter < numIter; iter++)
  {
    /// Create the operator that will do the work
    /// get the l2 norm
    Norm<2> l2Norm      (bulkData, &metaData.universal_part(), TURBO_NONE);
    Norm<2> l2Norm_turbo(bulkData, &metaData.universal_part(), turboOpt);

    //double TURBO_ELEMENT_time=0;
    TIME_IT2(l2Norm(sfx, sfx_res); , l2Norm_turbo(sfx, sfx_res_turbo);, "Should be the same turboOpt= ", turboOpt );
STKUNIT_UNIT_TEST(nodeRegistry, test_parallel_1_0)
{
  EXCEPTWATCH;
  MPI_Barrier( MPI_COMM_WORLD );

  // start_demo_nodeRegistry_test_parallel_1

  percept::PerceptMesh eMesh(3u);

  unsigned p_size = eMesh.get_parallel_size();
  unsigned p_rank = eMesh.get_rank();
  Util::setRank(eMesh.get_rank());

  eMesh.new_mesh(percept::GMeshSpec(std::string("1x1x")+toString(p_size)+std::string("|bbox:0,0,0,1,1,1")));

  // prepare for adding some quadratic elements
  mesh::Part& block_hex_20 = eMesh.get_fem_meta_data()->declare_part("block_hex_20", eMesh.element_rank());
  /// set cell topology for the part block_hex_20
  mesh::fem::set_cell_topology< shards::Hexahedron<20>  >( block_hex_20 );
  stk_classic::io::put_io_part_attribute(block_hex_20);

  eMesh.commit();
  eMesh.print_info();
  eMesh.save_as("./cube1x1x2_hex-20-orig.e");

  mesh::Part* block_hex_8 = const_cast<mesh::Part *>(eMesh.getPart("block_1"));

  NodeRegistry nodeRegistry(eMesh);
  nodeRegistry.initialize();

  if (p_size <= 2)
  {
    // pick an element on the processor boundary
    unsigned elem_num_local = 1;
    unsigned elem_num_ghost = 2;
    if (p_size == 1)
      elem_num_ghost = 1;

    stk_classic::mesh::Entity* element_local_p = eMesh.get_bulk_data()->get_entity(eMesh.element_rank(), elem_num_local);
    stk_classic::mesh::Entity* element_ghost_p = eMesh.get_bulk_data()->get_entity(eMesh.element_rank(), elem_num_ghost);
    if (p_rank == 1)
    {
      element_local_p = eMesh.get_bulk_data()->get_entity(eMesh.element_rank(), elem_num_ghost);
      element_ghost_p = eMesh.get_bulk_data()->get_entity(eMesh.element_rank(), elem_num_local);
    }

    dw() << "P["<<p_rank<<"] elem_num_local = " << elem_num_local << DWENDL;
    dw() << "P["<<p_rank<<"] elem_num_ghost = " << elem_num_ghost << DWENDL;

    stk_classic::mesh::Entity& element_local = *element_local_p;
    stk_classic::mesh::Entity& element_ghost = *element_ghost_p;

    std::cout << "P["<<p_rank<<"] element_local = " << element_local << std::endl;
    std::cout << "P["<<p_rank<<"] element_ghost = " << element_ghost << std::endl;

    // choose edges to be used for new node locations (i.e., this would model a serendipity-like element with only edge Lagrange nodes)
    stk_classic::mesh::EntityRank stk_mesh_Edge = 1;
    NeededEntityType needed_entity_rank( stk_mesh_Edge, 1u);
    std::vector<NeededEntityType> needed_entity_ranks(1, needed_entity_rank);

    /*
     * 1st of three steps to create and associate new nodes - register need for new nodes, then check if node is remote, then get
     *   from remote proc if necessary; finally, the local node database is ready to be queried
     *
     * The pattern is to begin the step, loop over all elements (including ghosts) and invoke the local operation
     * The method doForAllSubEntities is a utility for performing the operation on all the sub entities.
     * If more granularity is desired, the member functions can be invoked directly for a particular sub-entity.
     */
    nodeRegistry.beginRegistration();
    nodeRegistry.doForAllSubEntities(&NodeRegistry::registerNeedNewNode, element_local, needed_entity_ranks);
    nodeRegistry.doForAllSubEntities(&NodeRegistry::registerNeedNewNode, element_ghost, needed_entity_ranks);
    nodeRegistry.endRegistration();

    std::cout << "P["<<p_rank<<"] nodeRegistry size  = " << nodeRegistry.total_size() << std::endl;
    std::cout << "P["<<p_rank<<"] nodeRegistry lsize = " << nodeRegistry.local_size() << std::endl;

    dw() << "P["<<p_rank<<"] nodeRegistry size       = " << nodeRegistry.total_size() << DWENDL;
    dw() << "P["<<p_rank<<"] nodeRegistry lsize      = " << nodeRegistry.local_size() << DWENDL;

    // could do local create of elements here
    nodeRegistry.beginLocalMeshMods();
    nodeRegistry.endLocalMeshMods();

    // check if the newly requested nodes are local or remote
    nodeRegistry.beginCheckForRemote();
    nodeRegistry.doForAllSubEntities(&NodeRegistry::checkForRemote, element_local, needed_entity_ranks);
    nodeRegistry.doForAllSubEntities(&NodeRegistry::checkForRemote, element_ghost, needed_entity_ranks);
    nodeRegistry.endCheckForRemote();

    // get the new nodes from other procs if they are nonlocal
    nodeRegistry.beginGetFromRemote();
    nodeRegistry.doForAllSubEntities(&NodeRegistry::getFromRemote, element_local, needed_entity_ranks);
    nodeRegistry.doForAllSubEntities(&NodeRegistry::getFromRemote, element_ghost, needed_entity_ranks);
    nodeRegistry.endGetFromRemote();

    // now we can get the new node's id and entity
    unsigned iSubDimOrd = 4u;
    if (p_rank)
    {
      iSubDimOrd = 0u;
    }
    NodeIdsOnSubDimEntityType& nodeIds_onSE_0 = *( nodeRegistry.getNewNodesOnSubDimEntity(element_local, needed_entity_rank.first, iSubDimOrd));
    stk_classic::mesh::Entity*  node_0   = eMesh.get_bulk_data()->get_entity(stk_classic::mesh::fem::FEMMetaData::NODE_RANK, nodeIds_onSE_0[0]->identifier());

    // should be the same node on each proc
    std::cout << "P[" << p_rank << "] nodeId_0 = " << nodeIds_onSE_0 << " node_0= " << node_0 << std::endl;

    // end_demo

#if STK_ADAPT_HAVE_YAML_CPP
    if (p_size == 1)
      {
        if (1) {
          YAML::Emitter out;
          out << YAML::Anchor("NodeRegistry::map");
          out << YAML::BeginMap;
          out << YAML::Key << YAML::BeginSeq << 1 << 2 << YAML::EndSeq << YAML::Value << YAML::BeginSeq << -1 << -2 << YAML::EndSeq;
          out << YAML::Key << 1;
          out << YAML::Value << 2;
          out << YAML::Key << 3;
          out << YAML::Value << 4;
          out << YAML::EndMap;
          //std::cout << "out=\n" << out.c_str() << "\n=out" << std::endl;
          std::string expected_result = "&NodeRegistry::map\n?\n  - 1\n  - 2\n:\n  - -1\n  - -2\n1: 2\n3: 4";
          //std::cout << "out2=\n" << expected_result << std::endl;
          STKUNIT_EXPECT_TRUE(expected_result == std::string(out.c_str()));
        }

        YAML::Emitter yaml;
        std::cout << "\nnodeRegistry.serialize_write(yaml)" << std::endl;
        SerializeNodeRegistry::serialize_write(nodeRegistry, yaml, 0);
        //std::cout << yaml.c_str() << std::endl;
        if (!yaml.good())
          {
            std::cout << "Emitter error: " << yaml.good() << " " <<yaml.GetLastError() << "\n";
            STKUNIT_EXPECT_TRUE(false);
          }
        std::ofstream file1("out.yaml");
        file1 << yaml.c_str();
        file1.close();
        std::ifstream file2("out.yaml");
        YAML::Parser parser(file2);
        YAML::Node doc;

        try {
          while(parser.GetNextDocument(doc)) {
            std::cout << "\n read doc.Type() = " << doc.Type() << " doc.Tag()= " << doc.Tag() << " doc.size= " << doc.size() << std::endl;
            if (doc.Type() == YAML::NodeType::Map)
              {
                for(YAML::Iterator it=doc.begin();it!=doc.end();++it) {
                  int key, value;
                  std::cout << "read it.first().Type() = " << it.first().Type() << " it.first().Tag()= " << it.first().Tag() << std::endl;
                  std::cout << "read it.second().Type() = " << it.second().Type() << " it.second().Tag()= " << it.second().Tag() << std::endl;
                  const YAML::Node& keySeq = it.first();
                  for(YAML::Iterator itk=keySeq.begin();itk!=keySeq.end();++itk) {
                    *itk >> key;
                    std::cout << "read key= " << key << std::endl;
                  }
              
                  const YAML::Node& valSeq = it.second();
                  for(YAML::Iterator itv=valSeq.begin();itv!=valSeq.end();++itv) {
                    *itv >> value;
                    std::cout << "read value= " << value << std::endl;
                  }
              
                }
              }
          }
        }
        catch(YAML::ParserException& e) {
          std::cout << e.what() << "\n";
          STKUNIT_EXPECT_TRUE(false);
        }

        file2.close();
        std::ifstream file3("out.yaml");
        NodeRegistry nrNew(eMesh);
        SerializeNodeRegistry::serialize_read(nrNew, file3);
        YAML::Emitter yaml3;
        std::cout << "\nnrNew.serialize_write(yaml3)" << std::endl;
        SerializeNodeRegistry::serialize_write(nrNew, yaml3, 0);
        std::cout << yaml3.c_str() << std::endl;
        
        //exit(1);
      }
#endif
    // start_demo_nodeRegistry_test_parallel_1_quadratic_elem

    // change element to be a serendipity quadratic element
    eMesh.get_bulk_data()->modification_begin();

    //getCellTopologyData< shards::Node  >()
    const CellTopologyData *const cell_topo_data =stk_classic::percept::PerceptMesh::get_cell_topology(block_hex_20);
    CellTopology cell_topo(cell_topo_data);

    for (unsigned isd = 0; isd < 12; isd++)
    {
      nodeRegistry.makeCentroidCoords(element_local, needed_entity_rank.first, isd);
      NodeIdsOnSubDimEntityType& nodeIds_onSE_0_loc = *( nodeRegistry.getNewNodesOnSubDimEntity(element_local, needed_entity_rank.first, isd));

      stk_classic::mesh::Entity*  node   = eMesh.get_bulk_data()->get_entity(stk_classic::mesh::fem::FEMMetaData::NODE_RANK, nodeIds_onSE_0_loc[0]->identifier());

      unsigned edge_ord = 8u + isd;
      //unsigned n_edge_ord = cell_topo_data->edge[isd].topology->node_count;
      //std::cout << "n_edge_ord = " << n_edge_ord << std::endl;
      edge_ord = cell_topo_data->edge[isd].node[2];
      eMesh.get_bulk_data()->declare_relation(element_local, *node, edge_ord);
    }

    std::vector<stk_classic::mesh::Part*> add_parts(1, &block_hex_20);
    std::vector<stk_classic::mesh::Part*> remove_parts(1, block_hex_8);
    eMesh.get_bulk_data()->change_entity_parts( element_local, add_parts, remove_parts );

    eMesh.get_bulk_data()->modification_end();
    eMesh.print_info("After quadratic");

    eMesh.save_as("./cube1x1x2_hex-20.e");
    //exit(1);
  }