Example #1
0
    int FunctionSpaceTools_Test03(const bool verbose) {

      Teuchos::RCP<std::ostream> outStream;
      Teuchos::oblackholestream bhs; // outputs nothing

      if (verbose)
        outStream = Teuchos::rcp(&std::cout, false);
      else
        outStream = Teuchos::rcp(&bhs, false);

      Teuchos::oblackholestream oldFormatState;
      oldFormatState.copyfmt(std::cout);

      typedef typename
        Kokkos::Impl::is_space<DeviceSpaceType>::host_mirror_space::execution_space HostSpaceType ;

      *outStream << "DeviceSpace::  "; DeviceSpaceType::print_configuration(*outStream, false);
      *outStream << "HostSpace::    ";   HostSpaceType::print_configuration(*outStream, false);
      
      *outStream
        << "===============================================================================\n"
        << "|                                                                             |\n"
        << "|                      Unit Test (FunctionSpaceTools)                         |\n"
        << "|                                                                             |\n"
        << "|     1) Basic operator transformations and integration in HDIV:              |\n"
        << "|                                                                             |\n"
        << "|  Questions? Contact  Pavel Bochev ([email protected]) or                   |\n"
        << "|                      Denis Ridzal ([email protected]) or                   |\n"
        << "|                      Kyungjoo Kim ([email protected]).                      |\n"
        << "|                                                                             |\n"
        << "|  Intrepid's website: http://trilinos.sandia.gov/packages/intrepid           |\n"
        << "|  Trilinos website:   http://trilinos.sandia.gov                             |\n"
        << "|                                                                             |\n"
        << "===============================================================================\n";

      typedef CellTools<DeviceSpaceType> ct;      
      typedef FunctionSpaceTools<DeviceSpaceType> fst;      
      typedef Kokkos::DynRankView<ValueType,DeviceSpaceType> DynRankView;

      int errorFlag = 0;

      *outStream
        << "\n"
        << "===============================================================================\n"
        << "| TEST 1: correctness of math operations                                      |\n"
        << "===============================================================================\n";
      
      outStream->precision(20);
      
      try {
        DefaultCubatureFactory cub_factory;                            

        shards::CellTopology cell_topo = shards::getCellTopologyData< shards::Hexahedron<8> >();    

        const auto cub_degree = 20;                                                               
        auto cub = cub_factory.create<DeviceSpaceType,ValueType,ValueType>(cell_topo, cub_degree); 

        const auto space_dim = cub->getDimension();                                              
        const auto num_cub_points = cub->getNumPoints();                                          

        Basis_HDIV_HEX_I1_FEM<DeviceSpaceType> basis;                        
        const auto num_fields = basis.getCardinality();                                       
        
        /* Cell geometries and orientations. */
        const auto num_cells = 4;
        const auto num_nodes = 8;
        
        const ValueType hexnodes[] = {
          // hex 0  -- affine
          -1.0, -1.0, -1.0,
          1.0, -1.0, -1.0,
          1.0, 1.0, -1.0,
          -1.0, 1.0, -1.0,
          -1.0, -1.0, 1.0,
          1.0, -1.0, 1.0,
          1.0, 1.0, 1.0,
          -1.0, 1.0, 1.0,
          // hex 1  -- affine
          -3.0, -3.0, 1.0,
          6.0, 3.0, 1.0,
          7.0, 8.0, 0.0,
          -2.0, 2.0, 0.0,
          -3.0, -3.0, 4.0,
          6.0, 3.0, 4.0,
          7.0, 8.0, 3.0,
          -2.0, 2.0, 3.0,
          // hex 2  -- affine
          -3.0, -3.0, 0.0,
          9.0, 3.0, 0.0,
          15.0, 6.1, 0.0,
          3.0, 0.1, 0.0,
          9.0, 3.0, 0.1,
          21.0, 9.0, 0.1,
          27.0, 12.1, 0.1,
          15.0, 6.1, 0.1,
          // hex 3  -- nonaffine
          -2.0, -2.0, 0.0,
          2.0, -1.0, 0.0,
          1.0, 6.0, 0.0,
          -1.0, 1.0, 0.0,
          0.0, 0.0, 1.0,
          1.0, 0.0, 1.0,
          1.0, 1.0, 1.0,
          0.0, 1.0, 1.0
        };
        
        const ValueType facesigns[] = {
          1, 1, 1, 1, 1, 1,
          1, -1, 1, -1, 1, -1,
          -1, -1, 1, 1, -1, 1,
          -1, -1, 1, 1, -1, -1
        };
        
        /* Computational arrays. */
        DynRankView ConstructWithLabel( cub_points,  num_cub_points, space_dim);
        DynRankView ConstructWithLabel( cub_weights, num_cub_points);

        DynRankView ConstructWithLabel( cell_nodes,       num_cells, num_nodes, space_dim);
        DynRankView ConstructWithLabel( field_signs,      num_cells, num_fields);

        DynRankView ConstructWithLabel( jacobian,         num_cells, num_cub_points, space_dim, space_dim);
        DynRankView ConstructWithLabel( jacobian_det,     num_cells, num_cub_points);
        DynRankView ConstructWithLabel( weighted_measure, num_cells, num_cub_points);
        
        DynRankView ConstructWithLabel( div_of_basis_at_cub_points,                                 num_fields, num_cub_points);
        DynRankView ConstructWithLabel( transformed_div_of_basis_at_cub_points,          num_cells, num_fields, num_cub_points);
        DynRankView ConstructWithLabel( weighted_transformed_div_of_basis_at_cub_points, num_cells, num_fields, num_cub_points);
        DynRankView ConstructWithLabel( stiffness_matrices,                              num_cells, num_fields, num_fields);
        
        DynRankView ConstructWithLabel( value_of_basis_at_cub_points,                                 num_fields, num_cub_points, space_dim);
        DynRankView ConstructWithLabel( transformed_value_of_basis_at_cub_points,          num_cells, num_fields, num_cub_points, space_dim);
        DynRankView ConstructWithLabel( weighted_transformed_value_of_basis_at_cub_points, num_cells, num_fields, num_cub_points, space_dim);
        DynRankView ConstructWithLabel( mass_matrices,                                     num_cells, num_fields, num_fields);

        /******************* START COMPUTATION ***********************/

        // get cubature points and weights
        cub->getCubature(cub_points, cub_weights);

        const Kokkos::DynRankView<const ValueType,Kokkos::LayoutRight,Kokkos::HostSpace> cell_nodes_host (&hexnodes[0],  num_cells, num_nodes, space_dim); 
        const Kokkos::DynRankView<const ValueType,Kokkos::LayoutRight,Kokkos::HostSpace> field_signs_host(&facesigns[0], num_cells, num_fields); 
        
        Kokkos::deep_copy( cell_nodes,  cell_nodes_host  );
        Kokkos::deep_copy( field_signs, field_signs_host );
        
        // compute geometric cell information
        ct::setJacobian(jacobian, cub_points, cell_nodes, cell_topo);
        ct::setJacobianDet(jacobian_det, jacobian);

        // compute weighted measure
        fst::computeCellMeasure(weighted_measure, jacobian_det, cub_weights);

        // **Computing stiffness matrices:
        basis.getValues(div_of_basis_at_cub_points, cub_points, OPERATOR_DIV);
        
        // transform divergences of basis functions
        fst::HDIVtransformDIV(transformed_div_of_basis_at_cub_points,
                              jacobian_det,
                              div_of_basis_at_cub_points);
        
        // multiply with weighted measure
        fst::multiplyMeasure(weighted_transformed_div_of_basis_at_cub_points,
                             weighted_measure,
                             transformed_div_of_basis_at_cub_points);
        
        // we can apply the field signs to the basis function arrays, or after the fact, see below
        fst::applyFieldSigns(transformed_div_of_basis_at_cub_points, field_signs);
        fst::applyFieldSigns(weighted_transformed_div_of_basis_at_cub_points, field_signs);
        
        // compute stiffness matrices
        fst::integrate(stiffness_matrices,
                       transformed_div_of_basis_at_cub_points,
                       weighted_transformed_div_of_basis_at_cub_points);
        

        // **Computing mass matrices:
        basis.getValues(value_of_basis_at_cub_points, cub_points, OPERATOR_VALUE);

            
        
        // transform values of basis functions
        fst::HDIVtransformVALUE(transformed_value_of_basis_at_cub_points,
                                jacobian,
                                jacobian_det,
                                value_of_basis_at_cub_points);

        // multiply with weighted measure
        fst::multiplyMeasure(weighted_transformed_value_of_basis_at_cub_points,
                             weighted_measure,
                             transformed_value_of_basis_at_cub_points);
        
        // compute mass matrices
        fst::integrate(mass_matrices,
                       transformed_value_of_basis_at_cub_points,
                       weighted_transformed_value_of_basis_at_cub_points);
        
        // apply field signs
        fst::applyLeftFieldSigns(mass_matrices, field_signs);
        fst::applyRightFieldSigns(mass_matrices, field_signs);
        
        /*******************  STOP COMPUTATION ***********************/
        
        
        /******************* START COMPARISON ***********************/
        std::string basedir = "../testdata";
        for (auto cid=0;cid<num_cells-1;++cid) {
          std::stringstream namestream;
          std::string filename;
          namestream <<  basedir << "/mass_HDIV_HEX_I1_FEM" << "_" << "0" << cid+1 << ".dat";
          namestream >> filename;

          *outStream << "\nCell ID : " << cid << "  mass matrix comparing with " << filename << "\n\n";
          
          std::ifstream massfile(&filename[0]);
          if (massfile.is_open()) {
            const auto mass_matrix_cell = Kokkos::subdynrankview(mass_matrices, cid, Kokkos::ALL(), Kokkos::ALL());
            errorFlag += compareToAnalytic(massfile, 
                                           mass_matrix_cell, 
                                           1e-10,
                                           verbose);
            massfile.close();
          } else {
            errorFlag = -1;
            INTREPID2_TEST_FOR_EXCEPTION( true, std::runtime_error,
                                          "Failed to open a file" );
          }
          
          namestream.clear();
          namestream << basedir << "/stiff_HDIV_HEX_I1_FEM" << "_" << "0" << cid+1 << ".dat";
          namestream >> filename;

          *outStream << "\nCell ID : " << cid << "  stiffness matrix comparing with " << filename << "\n\n";
          
          std::ifstream stifffile(&filename[0]);
          if (stifffile.is_open()) {
            const auto stiffness_matrix_cell = Kokkos::subdynrankview(stiffness_matrices, cid, Kokkos::ALL(), Kokkos::ALL());
            errorFlag += compareToAnalytic(stifffile,
                                           stiffness_matrix_cell,
                                           1e-10,
                                           verbose);
            stifffile.close();
          } else {
            errorFlag = -1;
            INTREPID2_TEST_FOR_EXCEPTION( true, std::runtime_error,
                                          "Failed to open a file" );
          }
        }

        for (auto cid=3;cid<num_cells;++cid) {
          std::stringstream namestream;
          std::string filename;
          namestream <<  basedir << "/mass_fp_HDIV_HEX_I1_FEM" << "_" << "0" << cid+1 << ".dat";
          namestream >> filename;

          *outStream << "\nCell ID : " << cid << "  mass matrix comparing with " << filename << "\n\n";
          
          std::ifstream massfile(&filename[0]);
          if (massfile.is_open()) {
            const auto mass_matrix_cell = Kokkos::subdynrankview(mass_matrices, cid, Kokkos::ALL(), Kokkos::ALL());
            errorFlag += compareToAnalytic(massfile, 
                                           mass_matrix_cell, 
                                           1e-4, 
                                           verbose,
                                           INTREPID2_UTILS_SCALAR);
            massfile.close();
          } else {
            errorFlag = -1;
            INTREPID2_TEST_FOR_EXCEPTION( true, std::runtime_error,
                                          "Failed to open a file" );
          }
          
          namestream.clear();
          namestream << basedir << "/stiff_fp_HDIV_HEX_I1_FEM" << "_" << "0" << cid+1 << ".dat";
          namestream >> filename;
          
          *outStream << "\nCell ID : " << cid << "  stiffness matrix comparing with " << filename << "\n\n";

          std::ifstream stifffile(&filename[0]);
          if (stifffile.is_open()) {
            const auto stiffness_matrix_cell = Kokkos::subdynrankview(stiffness_matrices, cid, Kokkos::ALL(), Kokkos::ALL());
            errorFlag += compareToAnalytic(stifffile,
                                           stiffness_matrix_cell,
                                           1e-4, 
                                           verbose,
                                           INTREPID2_UTILS_SCALAR);
            stifffile.close();
          } else {
            errorFlag = -1;
            INTREPID2_TEST_FOR_EXCEPTION( true, std::runtime_error,
                                          "Failed to open a file" );
          }
        }
        /******************* STOP COMPARISON ***********************/

        *outStream << "\n";
      } catch (std::logic_error err) {
        *outStream << "UNEXPECTED ERROR !!! ----------------------------------------------------------\n";
        *outStream << err.what() << '\n';
        *outStream << "-------------------------------------------------------------------------------" << "\n\n";
        errorFlag = -1000;
      }

      if (errorFlag != 0)
        std::cout << "End Result: TEST FAILED\n";
      else
        std::cout << "End Result: TEST PASSED\n";
      
      // reset format state of std::cout
      std::cout.copyfmt(oldFormatState);

      return errorFlag;
    }
Example #2
0
int main(int argc, char *argv[]) {
  
  Teuchos::GlobalMPISession mpiSession(&argc, &argv);

  // This little trick lets us print to std::cout only if
  // a (dummy) command-line argument is provided.
  int iprint     = argc - 1;
  Teuchos::RCP<std::ostream> outStream;
  Teuchos::oblackholestream bhs; // outputs nothing
  if (iprint > 0)
    outStream = Teuchos::rcp(&std::cout, false);
  else
    outStream = Teuchos::rcp(&bhs, false);
  
  // Save the format state of the original std::cout.
  Teuchos::oblackholestream oldFormatState;
  oldFormatState.copyfmt(std::cout);
  
  *outStream \
    << "===============================================================================\n" \
    << "|                                                                             |\n" \
    << "|                 Unit Test (Basis_HDIV_HEX_I1_FEM)                           |\n" \
    << "|                                                                             |\n" \
    << "|     1) Conversion of Dof tags into Dof ordinals and back                    |\n" \
    << "|     2) Basis values for VALUE and DIV operators                             |\n" \
    << "|                                                                             |\n" \
    << "|  Questions? Contact  Pavel Bochev  ([email protected]),                    |\n" \
    << "|                      Denis Ridzal  ([email protected]),                    |\n" \
    << "|                      Kara Peterson ([email protected]).                    |\n" \
    << "|                                                                             |\n" \
    << "|  Intrepid's website: http://trilinos.sandia.gov/packages/intrepid           |\n" \
    << "|  Trilinos website:   http://trilinos.sandia.gov                             |\n" \
    << "|                                                                             |\n" \
    << "===============================================================================\n"\
    << "| TEST 1: Basis creation, exception testing                                   |\n"\
    << "===============================================================================\n";
  
  // Define basis and error flag
  Basis_HDIV_HEX_I1_FEM<double, FieldContainer<double> > hexBasis;
  int errorFlag = 0;

  // Initialize throw counter for exception testing
  int nException     = 0;
  int throwCounter   = 0;

  // Define array containing the 8 vertices of the reference HEX, its center and 6 face centers
  FieldContainer<double> hexNodes(15, 3);
  hexNodes(0,0) = -1.0;  hexNodes(0,1) = -1.0;  hexNodes(0,2) = -1.0;
  hexNodes(1,0) =  1.0;  hexNodes(1,1) = -1.0;  hexNodes(1,2) = -1.0;
  hexNodes(2,0) =  1.0;  hexNodes(2,1) =  1.0;  hexNodes(2,2) = -1.0;
  hexNodes(3,0) = -1.0;  hexNodes(3,1) =  1.0;  hexNodes(3,2) = -1.0;
  
  hexNodes(4,0) = -1.0;  hexNodes(4,1) = -1.0;  hexNodes(4,2) =  1.0;
  hexNodes(5,0) =  1.0;  hexNodes(5,1) = -1.0;  hexNodes(5,2) =  1.0;
  hexNodes(6,0) =  1.0;  hexNodes(6,1) =  1.0;  hexNodes(6,2) =  1.0;
  hexNodes(7,0) = -1.0;  hexNodes(7,1) =  1.0;  hexNodes(7,2) =  1.0;  
  
  hexNodes(8,0) =  0.0;  hexNodes(8,1) =  0.0;  hexNodes(8,2) =  0.0;
  
  hexNodes(9,0) =  1.0;  hexNodes(9,1) =  0.0;  hexNodes(9,2) =  0.0;
  hexNodes(10,0)= -1.0;  hexNodes(10,1)=  0.0;  hexNodes(10,2)=  0.0;
  
  hexNodes(11,0)=  0.0;  hexNodes(11,1)=  1.0;  hexNodes(11,2)=  0.0;
  hexNodes(12,0)=  0.0;  hexNodes(12,1)= -1.0;  hexNodes(12,2)=  0.0;
  
  hexNodes(13,0)=  0.0;  hexNodes(13,1)=  0.0;  hexNodes(13,2)=  1.0;
  hexNodes(14,0)=  0.0;  hexNodes(14,1)=  0.0;  hexNodes(14,2)= -1.0;

  
  // Generic array for the output values; needs to be properly resized depending on the operator type
  FieldContainer<double> vals;

  try{
    // exception #1: GRAD cannot be applied to HDIV functions 
    // resize vals to rank-3 container with dimensions (num. basis functions, num. points, arbitrary)
    vals.resize(hexBasis.getCardinality(), hexNodes.dimension(0), 3 );
    INTREPID_TEST_COMMAND( hexBasis.getValues(vals, hexNodes, OPERATOR_GRAD), throwCounter, nException );

    // exception #2: CURL cannot be applied to HDIV functions
    INTREPID_TEST_COMMAND( hexBasis.getValues(vals, hexNodes, OPERATOR_CURL), throwCounter, nException );

    // Exceptions 3-7: all bf tags/bf Ids below are wrong and should cause getDofOrdinal() and 
    // getDofTag() to access invalid array elements thereby causing bounds check exception
    // exception #3
    INTREPID_TEST_COMMAND( hexBasis.getDofOrdinal(3,0,0), throwCounter, nException );
    // exception #4
    INTREPID_TEST_COMMAND( hexBasis.getDofOrdinal(1,1,1), throwCounter, nException );
    // exception #5
    INTREPID_TEST_COMMAND( hexBasis.getDofOrdinal(0,4,1), throwCounter, nException );
    // exception #6
    INTREPID_TEST_COMMAND( hexBasis.getDofTag(12), throwCounter, nException );
    // exception #7
    INTREPID_TEST_COMMAND( hexBasis.getDofTag(-1), throwCounter, nException );

#ifdef HAVE_INTREPID_DEBUG
    // Exceptions 8- test exception handling with incorrectly dimensioned input/output arrays
    // exception #8: input points array must be of rank-2
    FieldContainer<double> badPoints1(4, 5, 3);
    INTREPID_TEST_COMMAND( hexBasis.getValues(vals, badPoints1, OPERATOR_VALUE), throwCounter, nException );
    
    // exception #9 dimension 1 in the input point array must equal space dimension of the cell
    FieldContainer<double> badPoints2(4, 2);
    INTREPID_TEST_COMMAND( hexBasis.getValues(vals, badPoints2, OPERATOR_VALUE), throwCounter, nException );
    
    // exception #10 output values must be of rank-3 for OPERATOR_VALUE
    FieldContainer<double> badVals1(4, 3);
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals1, hexNodes, OPERATOR_VALUE), throwCounter, nException );
 
    // exception #11 output values must be of rank-2 for OPERATOR_DIV
    FieldContainer<double> badVals2(4, 3, 3);
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals2, hexNodes, OPERATOR_DIV), throwCounter, nException );
    
    // exception #12 incorrect 0th dimension of output array (must equal number of basis functions)
    FieldContainer<double> badVals3(hexBasis.getCardinality() + 1, hexNodes.dimension(0), 3);
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals3, hexNodes, OPERATOR_VALUE), throwCounter, nException );
    
    // exception #13 incorrect 0th dimension of output array (must equal number of basis functions)
    FieldContainer<double> badVals4(hexBasis.getCardinality() + 1, hexNodes.dimension(0));
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals4, hexNodes, OPERATOR_DIV), throwCounter, nException );

    // exception #14 incorrect 1st dimension of output array (must equal number of points)
    FieldContainer<double> badVals5(hexBasis.getCardinality(), hexNodes.dimension(0) + 1, 3);
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals5, hexNodes, OPERATOR_VALUE), throwCounter, nException );

    // exception #15 incorrect 1st dimension of output array (must equal number of points)
    FieldContainer<double> badVals6(hexBasis.getCardinality(), hexNodes.dimension(0) + 1);
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals6, hexNodes, OPERATOR_DIV), throwCounter, nException );

    // exception #16: incorrect 2nd dimension of output array (must equal the space dimension)
    FieldContainer<double> badVals7(hexBasis.getCardinality(), hexNodes.dimension(0), 4);
    INTREPID_TEST_COMMAND( hexBasis.getValues(badVals7, hexNodes, OPERATOR_VALUE), throwCounter, nException );
#endif
    
  }
  catch (std::logic_error err) {
    *outStream << "UNEXPECTED ERROR !!! ----------------------------------------------------------\n";
    *outStream << err.what() << '\n';
    *outStream << "-------------------------------------------------------------------------------" << "\n\n";
    errorFlag = -1000;
  };
  
  // Check if number of thrown exceptions matches the one we expect 
  if (throwCounter != nException) {
    errorFlag++;
    *outStream << std::setw(70) << "^^^^----FAILURE!" << "\n";
  }
  
  *outStream \
    << "\n"
    << "===============================================================================\n"\
    << "| TEST 2: correctness of tag to enum and enum to tag lookups                  |\n"\
    << "===============================================================================\n";
  
  try{
    std::vector<std::vector<int> > allTags = hexBasis.getAllDofTags();
    
    // Loop over all tags, lookup the associated dof enumeration and then lookup the tag again
    for (unsigned i = 0; i < allTags.size(); i++) {
      int bfOrd  = hexBasis.getDofOrdinal(allTags[i][0], allTags[i][1], allTags[i][2]);
      
      std::vector<int> myTag = hexBasis.getDofTag(bfOrd);
       if( !( (myTag[0] == allTags[i][0]) &&
              (myTag[1] == allTags[i][1]) &&
              (myTag[2] == allTags[i][2]) &&
              (myTag[3] == allTags[i][3]) ) ) {
        errorFlag++;
        *outStream << std::setw(70) << "^^^^----FAILURE!" << "\n";
        *outStream << " getDofOrdinal( {" 
          << allTags[i][0] << ", " 
          << allTags[i][1] << ", " 
          << allTags[i][2] << ", " 
          << allTags[i][3] << "}) = " << bfOrd <<" but \n";   
        *outStream << " getDofTag(" << bfOrd << ") = { "
          << myTag[0] << ", " 
          << myTag[1] << ", " 
          << myTag[2] << ", " 
          << myTag[3] << "}\n";        
      }
    }
    
    // Now do the same but loop over basis functions
    for( int bfOrd = 0; bfOrd < hexBasis.getCardinality(); bfOrd++) {
      std::vector<int> myTag  = hexBasis.getDofTag(bfOrd);
      int myBfOrd = hexBasis.getDofOrdinal(myTag[0], myTag[1], myTag[2]);
      if( bfOrd != myBfOrd) {
        errorFlag++;
        *outStream << std::setw(70) << "^^^^----FAILURE!" << "\n";
        *outStream << " getDofTag(" << bfOrd << ") = { "
          << myTag[0] << ", " 
          << myTag[1] << ", " 
          << myTag[2] << ", " 
          << myTag[3] << "} but getDofOrdinal({" 
          << myTag[0] << ", " 
          << myTag[1] << ", " 
          << myTag[2] << ", " 
          << myTag[3] << "} ) = " << myBfOrd << "\n";
      }
    }
  }
  catch (std::logic_error err){
    *outStream << err.what() << "\n\n";
    errorFlag = -1000;
  };
  
  *outStream \
    << "\n"
    << "===============================================================================\n"\
    << "| TEST 3: correctness of basis function values                                |\n"\
    << "===============================================================================\n";
  
  outStream -> precision(20);
  
  // VALUE: Each row pair gives the 6x3 correct basis set values at an evaluation point: (P,F,D) layout
  double basisValues[] = {
    // bottom 4 vertices
      0.,-0.25,0.,  0.,0.,0.,  0.,0.,0., -0.25,0.,0.,  0.,0.,-0.25,  0.,0.,0.,
      0.,-0.25,0.,  0.25,0.,0.,  0.,0.,0.,  0.,0.,0.,  0.,0.,-0.25,  0.,0.,0.,  
      0.,0.,0.,  0.25,0.,0.,  0.,0.25,0.,  0.,0.,0.,  0.,0.,-0.25,  0.,0.,0.,
      0.,0.,0.,  0.,0.,0.,  0.,0.25,0., -0.25,0.,0.,  0.,0.,-0.25,  0.,0.,0.,
    // top 4 vertices
      0.,-0.25,0.,  0.,0.,0.,  0.,0.,0.,  -0.25,0.,0.,  0.,0.,0.,  0.,0.,0.25,
      0.,-0.25,0.,  0.25,0.,0.,  0.,0.,0.,  0.,0.,0.,  0.,0.,0.,  0.,0.,0.25,
      0.,0.,0.,  0.25,0.,0.,  0.,0.25,0.,  0.,0.,0.,  0.,0.,0.,  0.,0.,0.25,
      0.,0.,0.,  0.,0.,0.,  0.,0.25,0.,  -0.25,0.,0.,  0.,0.,0.,  0.,0.,0.25,
    // center {0, 0, 0}
      0.,-0.125,0.,  0.125,0.,0.,  0.,0.125,0.,  -0.125,0.,0.,  0.,0.,-0.125,  0.,0.,0.125,
    // faces { 1, 0, 0} and {-1, 0, 0}
      0.,-0.125,0.,  0.25,0.,0.,  0.,0.125,0.,  0.,0.,0.,  0.,0.,-0.125,  0.,0.,0.125, 
      0.,-0.125,0.,  0.,0.,0.,  0.,0.125,0., -0.25,0.,0.,  0.,0.,-0.125,  0.,0.,0.125,
    // faces { 0, 1, 0} and { 0,-1, 0}
      0.,0.,0.,  0.125,0.,0.,  0.,0.25,0.,  -0.125,0.,0.,  0.,0.,-0.125,  0.,0.,0.125,
      0.,-0.25,0.,  0.125,0.,0.,  0.,0.,0.,  -0.125,0.,0.,  0.,0.,-0.125,  0.,0.,0.125,
    // faces {0, 0, 1} and {0, 0, -1}
      0.,-0.125,0.,  0.125,0.,0.,  0.,0.125,0.,  -0.125,0.,0.,  0.,0.,0.,  0.,0.,0.25,
      0.,-0.125,0.,  0.125,0.,0.,  0.,0.125,0.,  -0.125,0.,0.,  0.,0.,-0.25,  0.,0.,0.
  };
  
  // DIV: each row gives the 6 correct values of the divergence of the 6 basis functions: (P,F) layout
  double basisDivs[] = {   
    // bottom 4 vertices
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
    // top 4 vertices
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
    // center {0, 0, 0}
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
    // faces { 1, 0, 0} and {-1, 0, 0}
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
    // faces { 0, 1, 0} and { 0,-1, 0}
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
    // faces {0, 0, 1} and {0, 0, -1}
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
      0.125, 0.125, 0.125, 0.125, 0.125, 0.125,  
  };
  
  try{
        
    // Dimensions for the output arrays:
    int numPoints = hexNodes.dimension(0);
    int numFields = hexBasis.getCardinality();
    int spaceDim  = hexBasis.getBaseCellTopology().getDimension();
    
    // Generic array for values and curls that will be properly sized before each call
    FieldContainer<double> vals;
    
    // Check VALUE of basis functions: resize vals to rank-3 container:
    vals.resize(numFields, numPoints, spaceDim);
    hexBasis.getValues(vals, hexNodes, OPERATOR_VALUE);
    for (int i = 0; i < numFields; i++) {
      for (int j = 0; j < numPoints; j++) {
        for (int k = 0; k < spaceDim; k++) {

          // compute offset for (P,F,D) data layout: indices are P->j, F->i, D->k
           int l = k + i * spaceDim + j * spaceDim * numFields;
           if (std::abs(vals(i,j,k) - basisValues[l]) > INTREPID_TOL) {
             errorFlag++;
             *outStream << std::setw(70) << "^^^^----FAILURE!" << "\n";

             // Output the multi-index of the value where the error is:
             *outStream << " At multi-index { ";
             *outStream << i << " ";*outStream << j << " ";*outStream << k << " ";
             *outStream << "}  computed value: " << vals(i,j,k)
               << " but reference value: " << basisValues[l] << "\n";
            }
         }
      }
    }

    // Check DIV of basis function: resize vals to rank-2 container
    vals.resize(numFields, numPoints);
    hexBasis.getValues(vals, hexNodes, OPERATOR_DIV);
    for (int i = 0; i < numFields; i++) {
      for (int j = 0; j < numPoints; j++) {
          int l =  i + j * numFields;
           if (std::abs(vals(i,j) - basisDivs[l]) > INTREPID_TOL) {
             errorFlag++;
             *outStream << std::setw(70) << "^^^^----FAILURE!" << "\n";

             // Output the multi-index of the value where the error is:
             *outStream << " At multi-index { ";
             *outStream << i << " ";*outStream << j << " ";
             *outStream << "}  computed divergence component: " << vals(i,j)
               << " but reference divergence component: " << basisDivs[l] << "\n";
         }
      }
    }
    
   }    
  
  // Catch unexpected errors
  catch (std::logic_error err) {
    *outStream << err.what() << "\n\n";
    errorFlag = -1000;
  };
  
  if (errorFlag != 0)
    std::cout << "End Result: TEST FAILED\n";
  else
    std::cout << "End Result: TEST PASSED\n";
  
  // reset format state of std::cout
  std::cout.copyfmt(oldFormatState);
  
  return errorFlag;
}
Example #3
0
int main(int argc, char *argv[]) {

  Teuchos::GlobalMPISession mpiSession(&argc, &argv);
 Kokkos::initialize();
  // This little trick lets us print to std::cout only if
  // a (dummy) command-line argument is provided.
  int iprint     = argc - 1;
  Teuchos::RCP<std::ostream> outStream;
  Teuchos::oblackholestream bhs; // outputs nothing
  if (iprint > 0)
    outStream = Teuchos::rcp(&std::cout, false);
  else
    outStream = Teuchos::rcp(&bhs, false);

  // Save the format state of the original std::cout.
  Teuchos::oblackholestream oldFormatState;
  oldFormatState.copyfmt(std::cout);

  *outStream \
    << "===============================================================================\n" \
    << "|                                                                             |\n" \
    << "|                      Unit Test (FunctionSpaceTools)                         |\n" \
    << "|                                                                             |\n" \
    << "|     1) Basic operator transformations and integration in HDIV:              |\n" \
    << "|                                                                             |\n" \
    << "|  Questions? Contact  Pavel Bochev ([email protected]) or                   |\n" \
    << "|                      Denis Ridzal ([email protected]).                     |\n" \
    << "|                                                                             |\n" \
    << "|  Intrepid's website: http://trilinos.sandia.gov/packages/intrepid           |\n" \
    << "|  Trilinos website:   http://trilinos.sandia.gov                             |\n" \
    << "|                                                                             |\n" \
    << "===============================================================================\n";
  

  int errorFlag = 0;

  typedef FunctionSpaceTools fst; 

  *outStream \
  << "\n"
  << "===============================================================================\n"\
  << "| TEST 1: correctness of math operations                                      |\n"\
  << "===============================================================================\n";

  outStream->precision(20);

  try {
    
      shards::CellTopology cellType = shards::getCellTopologyData< shards::Hexahedron<> >();    // cell type: hex

      /* Related to cubature. */
      DefaultCubatureFactory<double> cubFactory;                                                // create cubature factory
      int cubDegree = 20;                                                                       // cubature degree
      Teuchos::RCP<Cubature<double> > myCub = cubFactory.create(cellType, cubDegree);           // create default cubature
      int spaceDim = myCub->getDimension();                                                     // get spatial dimension 
      int numCubPoints = myCub->getNumPoints();                                                 // get number of cubature points

      /* Related to basis. */
      Basis_HDIV_HEX_I1_FEM<double, FieldContainer<double> > hexBasis;                          // create H-div basis on a hex
      int numFields = hexBasis.getCardinality();                                                // get basis cardinality
 
      /* Cell geometries and orientations. */
      int numCells    = 4;
      int numNodes    = 8;
      int numCellData = numCells*numNodes*spaceDim;
      int numSignData = numCells*numFields;

      double hexnodes[] = {
        // hex 0  -- affine
        -1.0, -1.0, -1.0,
        1.0, -1.0, -1.0,
        1.0, 1.0, -1.0,
        -1.0, 1.0, -1.0,
        -1.0, -1.0, 1.0,
        1.0, -1.0, 1.0,
        1.0, 1.0, 1.0,
        -1.0, 1.0, 1.0,
        // hex 1  -- affine
        -3.0, -3.0, 1.0,
        6.0, 3.0, 1.0,
        7.0, 8.0, 0.0,
        -2.0, 2.0, 0.0,
        -3.0, -3.0, 4.0,
        6.0, 3.0, 4.0,
        7.0, 8.0, 3.0,
        -2.0, 2.0, 3.0,
        // hex 2  -- affine
        -3.0, -3.0, 0.0,
        9.0, 3.0, 0.0,
        15.0, 6.1, 0.0,
        3.0, 0.1, 0.0,
        9.0, 3.0, 0.1,
        21.0, 9.0, 0.1,
        27.0, 12.1, 0.1,
        15.0, 6.1, 0.1,
        // hex 3  -- nonaffine
        -2.0, -2.0, 0.0,
        2.0, -1.0, 0.0,
        1.0, 6.0, 0.0,
        -1.0, 1.0, 0.0,
        0.0, 0.0, 1.0,
        1.0, 0.0, 1.0,
        1.0, 1.0, 1.0,
        0.0, 1.0, 1.0
      };

      short facesigns[] = {
        1, 1, 1, 1, 1, 1,
        1, -1, 1, -1, 1, -1,
        -1, -1, 1, 1, -1, 1,
        -1, -1, 1, 1, -1, -1
      };

      /* Computational arrays. */
      FieldContainer<double> cub_points(numCubPoints, spaceDim);
      FieldContainer<double> cub_weights(numCubPoints);
      FieldContainer<double> cell_nodes(numCells, numNodes, spaceDim);
      FieldContainer<short>  field_signs(numCells, numFields);
      FieldContainer<double> jacobian(numCells, numCubPoints, spaceDim, spaceDim);
      //FieldContainer<double> jacobian_inv(numCells, numCubPoints, spaceDim, spaceDim);
      FieldContainer<double> jacobian_det(numCells, numCubPoints);
      FieldContainer<double> weighted_measure(numCells, numCubPoints);

      FieldContainer<double> div_of_basis_at_cub_points(numFields, numCubPoints);
      FieldContainer<double> transformed_div_of_basis_at_cub_points(numCells, numFields, numCubPoints);
      FieldContainer<double> weighted_transformed_div_of_basis_at_cub_points(numCells, numFields, numCubPoints);
      FieldContainer<double> stiffness_matrices(numCells, numFields, numFields);

      FieldContainer<double> value_of_basis_at_cub_points(numFields, numCubPoints, spaceDim);
      FieldContainer<double> transformed_value_of_basis_at_cub_points(numCells, numFields, numCubPoints, spaceDim);
      FieldContainer<double> weighted_transformed_value_of_basis_at_cub_points(numCells, numFields, numCubPoints, spaceDim);
      FieldContainer<double> mass_matrices(numCells, numFields, numFields);

      /******************* START COMPUTATION ***********************/

      // get cubature points and weights
      myCub->getCubature(cub_points, cub_weights);

      // fill cell vertex array
      cell_nodes.setValues(hexnodes, numCellData);

      // set basis function signs, for each cell
      field_signs.setValues(facesigns, numSignData);

      // compute geometric cell information
      CellTools<double>::setJacobian(jacobian, cub_points, cell_nodes, cellType);
      //CellTools<double>::setJacobianInv(jacobian_inv, jacobian);
      CellTools<double>::setJacobianDet(jacobian_det, jacobian);

      // compute weighted measure
      fst::computeCellMeasure<double>(weighted_measure, jacobian_det, cub_weights);

      // Computing stiffness matrices:
      // tabulate divergences of basis functions at (reference) cubature points
      hexBasis.getValues(div_of_basis_at_cub_points, cub_points, OPERATOR_DIV);

      // transform divergences of basis functions
      fst::HDIVtransformDIV<double>(transformed_div_of_basis_at_cub_points,
                                    jacobian_det,
                                    div_of_basis_at_cub_points);

      // multiply with weighted measure
      fst::multiplyMeasure<double>(weighted_transformed_div_of_basis_at_cub_points,
                                   weighted_measure,
                                   transformed_div_of_basis_at_cub_points);

      // we can apply the field signs to the basis function arrays, or after the fact, see below
      fst::applyFieldSigns<double>(transformed_div_of_basis_at_cub_points, field_signs);
      fst::applyFieldSigns<double>(weighted_transformed_div_of_basis_at_cub_points, field_signs);

      // compute stiffness matrices
      fst::integrate<double>(stiffness_matrices,
                             transformed_div_of_basis_at_cub_points,
                             weighted_transformed_div_of_basis_at_cub_points,
                             COMP_CPP);

      // Computing mass matrices:
      // tabulate values of basis functions at (reference) cubature points
      hexBasis.getValues(value_of_basis_at_cub_points, cub_points, OPERATOR_VALUE);

      // transform values of basis functions
      fst::HDIVtransformVALUE<double>(transformed_value_of_basis_at_cub_points,
                                      jacobian,
                                      jacobian_det,
                                      value_of_basis_at_cub_points);

      // multiply with weighted measure
      fst::multiplyMeasure<double>(weighted_transformed_value_of_basis_at_cub_points,
                                   weighted_measure,
                                   transformed_value_of_basis_at_cub_points);

      // compute mass matrices
      fst::integrate<double>(mass_matrices,
                             transformed_value_of_basis_at_cub_points,
                             weighted_transformed_value_of_basis_at_cub_points,
                             COMP_CPP);

      // apply field signs
      fst::applyLeftFieldSigns<double>(mass_matrices, field_signs);
      fst::applyRightFieldSigns<double>(mass_matrices, field_signs);

      /*******************  STOP COMPUTATION ***********************/


      /******************* START COMPARISON ***********************/
      string basedir = "./testdata";
      for (int cell_id = 0; cell_id < numCells-1; cell_id++) {

        stringstream namestream;
        string filename;
        namestream <<  basedir << "/mass_HDIV_HEX_I1_FEM" << "_" << "0" << cell_id+1 << ".dat";
        namestream >> filename;

        ifstream massfile(&filename[0]);
        if (massfile.is_open()) {
          if (compareToAnalytic<double>(&mass_matrices(cell_id, 0, 0), massfile, 1e-10, iprint) > 0)
            errorFlag++;
          massfile.close();
        }
        else {
          errorFlag = -1;
          std::cout << "End Result: TEST FAILED\n";
          return errorFlag;
        }

        namestream.clear();
        namestream << basedir << "/stiff_HDIV_HEX_I1_FEM" << "_" << "0" << cell_id+1 << ".dat";
        namestream >> filename;

        ifstream stifffile(&filename[0]);
        if (stifffile.is_open())
        {
          if (compareToAnalytic<double>(&stiffness_matrices(cell_id, 0, 0), stifffile, 1e-10, iprint) > 0)
            errorFlag++;
          stifffile.close();
        }
        else {
          errorFlag = -1;
          std::cout << "End Result: TEST FAILED\n";
          return errorFlag;
        }

      }
      for (int cell_id = 3; cell_id < numCells; cell_id++) {

        stringstream namestream;
        string filename;
        namestream <<  basedir << "/mass_fp_HDIV_HEX_I1_FEM" << "_" << "0" << cell_id+1 << ".dat";
        namestream >> filename;

        ifstream massfile(&filename[0]);
        if (massfile.is_open()) {
          if (compareToAnalytic<double>(&mass_matrices(cell_id, 0, 0), massfile, 1e-4, iprint, INTREPID2_UTILS_SCALAR) > 0)
            errorFlag++;
          massfile.close();
        }
        else {
          errorFlag = -1;
          std::cout << "End Result: TEST FAILED\n";
          return errorFlag;
        }

        namestream.clear();
        namestream << basedir << "/stiff_fp_HDIV_HEX_I1_FEM" << "_" << "0" << cell_id+1 << ".dat";
        namestream >> filename;

        ifstream stifffile(&filename[0]);
        if (stifffile.is_open())
        {
          if (compareToAnalytic<double>(&stiffness_matrices(cell_id, 0, 0), stifffile, 1e-4, iprint, INTREPID2_UTILS_SCALAR) > 0)
            errorFlag++;
          stifffile.close();
        }
        else {
          errorFlag = -1;
          std::cout << "End Result: TEST FAILED\n";
          return errorFlag;
        }

      }
      /******************* STOP COMPARISON ***********************/

      *outStream << "\n";
  }// try Basis_HDIV_HEX_I1
  catch (std::logic_error err) {
    *outStream << "UNEXPECTED ERROR !!! ----------------------------------------------------------\n";
    *outStream << err.what() << '\n';
    *outStream << "-------------------------------------------------------------------------------" << "\n\n";
    errorFlag = -1000;
  };


  if (errorFlag != 0)
    std::cout << "End Result: TEST FAILED\n";
  else
    std::cout << "End Result: TEST PASSED\n";

  // reset format state of std::cout
  std::cout.copyfmt(oldFormatState);
 Kokkos::finalize();
  return errorFlag;
}
Example #4
0
int main(int argc, char *argv[]) {

   //Check number of arguments
   if (argc < 13) {
      std::cout <<"\n>>> ERROR: Invalid number of arguments.\n\n";
      std::cout <<"Usage:\n\n";
      std::cout <<"  ./Intrepid_example_Drivers_Example_02.exe NX NY NZ randomMesh mu1 mu2 mu1LX mu1RX mu1LY mu1RY mu1LZ mu1RZ verbose\n\n";
      std::cout <<" where \n";
      std::cout <<"   int NX              - num intervals in x direction (assumed box domain, -1,1) \n";
      std::cout <<"   int NY              - num intervals in y direction (assumed box domain, -1,1) \n";
      std::cout <<"   int NZ              - num intervals in z direction (assumed box domain, -1,1) \n";
      std::cout <<"   int randomMesh      - 1 if mesh randomizer is to be used 0 if not \n";
      std::cout <<"   double mu1          - material property value for region 1 \n";
      std::cout <<"   double mu2          - material property value for region 2 \n";
      std::cout <<"   double mu1LX        - left X boundary for region 1 \n";
      std::cout <<"   double mu1RX        - right X boundary for region 1 \n";
      std::cout <<"   double mu1LY        - left Y boundary for region 1 \n";
      std::cout <<"   double mu1RY        - right Y boundary for region 1 \n";
      std::cout <<"   double mu1LZ        - bottom Z boundary for region 1 \n";
      std::cout <<"   double mu1RZ        - top Z boundary for region 1 \n";
      std::cout <<"   verbose (optional)  - any character, indicates verbose output \n\n";
      exit(1);
   }

  // This little trick lets us print to std::cout only if
  // a (dummy) command-line argument is provided.
  int iprint     = argc - 1;
  Teuchos::RCP<std::ostream> outStream;
  Teuchos::oblackholestream bhs; // outputs nothing
  if (iprint > 12)
    outStream = Teuchos::rcp(&std::cout, false);
  else
    outStream = Teuchos::rcp(&bhs, false);
  
  // Save the format state of the original std::cout.
  Teuchos::oblackholestream oldFormatState;
  oldFormatState.copyfmt(std::cout);
  
  *outStream \
    << "===============================================================================\n" \
    << "|                                                                             |\n" \
    << "|   Example: Generate Mass and Stiffness Matrices and Right-Hand Side Vector  |\n"
    << "|     for Div-Curl System on Hexahedral Mesh with Div-Conforming Elements     |\n" \
    << "|                                                                             |\n" \
    << "|  Questions? Contact  Pavel Bochev  ([email protected]),                    |\n" \
    << "|                      Denis Ridzal  ([email protected]),                    |\n" \
    << "|                      Kara Peterson ([email protected]).                    |\n" \
    << "|                                                                             |\n" \
    << "|  Intrepid's website: http://trilinos.sandia.gov/packages/intrepid           |\n" \
    << "|  Trilinos website:   http://trilinos.sandia.gov                             |\n" \
    << "|                                                                             |\n" \
    << "===============================================================================\n";


// ************************************ GET INPUTS **************************************

  /* In the implementation for discontinuous material properties only the boundaries for
     region 1, associated with mu1, are input. The remainder of the grid is assumed to use mu2.
     Note that the material properties are assigned using the undeformed grid. */

    int NX            = atoi(argv[1]);  // num intervals in x direction (assumed box domain, -1,1)
    int NY            = atoi(argv[2]);  // num intervals in y direction (assumed box domain, -1,1)
    int NZ            = atoi(argv[3]);  // num intervals in z direction (assumed box domain, -1,1)
    int randomMesh    = atoi(argv[4]);  // 1 if mesh randomizer is to be used 0 if not
    double mu1        = atof(argv[5]);  // material property value for region 1
    double mu2        = atof(argv[6]);  // material property value for region 2
    double mu1LeftX   = atof(argv[7]);  // left X boundary for region 1
    double mu1RightX  = atof(argv[8]);  // right X boundary for region 1
    double mu1LeftY   = atof(argv[9]);  // left Y boundary for region 1
    double mu1RightY  = atof(argv[10]); // right Y boundary for region 1
    double mu1LeftZ   = atof(argv[11]); // left Z boundary for region 1
    double mu1RightZ  = atof(argv[12]); // right Z boundary for region 1

// *********************************** CELL TOPOLOGY **********************************

   // Get cell topology for base hexahedron
    typedef shards::CellTopology    CellTopology;
    CellTopology hex_8(shards::getCellTopologyData<shards::Hexahedron<8> >() );

   // Get dimensions 
    int numNodesPerElem = hex_8.getNodeCount();
    int numEdgesPerElem = hex_8.getEdgeCount();
    int numFacesPerElem = hex_8.getSideCount();
    int numNodesPerEdge = 2;
    int numNodesPerFace = 4;
    int numEdgesPerFace = 4;
    int spaceDim = hex_8.getDimension();

   // Build reference element edge to node map
    FieldContainer<int> refEdgeToNode(numEdgesPerElem,numNodesPerEdge);
    for (int i=0; i<numEdgesPerElem; i++){
        refEdgeToNode(i,0)=hex_8.getNodeMap(1, i, 0);
        refEdgeToNode(i,1)=hex_8.getNodeMap(1, i, 1);
    }

   // Build reference element face to node map
    FieldContainer<int> refFaceToNode(numFacesPerElem,numNodesPerFace);
    for (int i=0; i<numFacesPerElem; i++){
        refFaceToNode(i,0)=hex_8.getNodeMap(2, i, 0);
        refFaceToNode(i,1)=hex_8.getNodeMap(2, i, 1);
        refFaceToNode(i,2)=hex_8.getNodeMap(2, i, 2);
        refFaceToNode(i,3)=hex_8.getNodeMap(2, i, 3);
    }

// *********************************** GENERATE MESH ************************************

    *outStream << "Generating mesh ... \n\n";

    *outStream << "    NX" << "   NY" << "   NZ\n";
    *outStream << std::setw(5) << NX <<
                 std::setw(5) << NY <<
                 std::setw(5) << NZ << "\n\n";

   // Print mesh information
    int numElems = NX*NY*NZ;
    int numNodes = (NX+1)*(NY+1)*(NZ+1);
    int numEdges = (NX)*(NY + 1)*(NZ + 1) + (NX + 1)*(NY)*(NZ + 1) + (NX + 1)*(NY + 1)*(NZ);
    int numFaces = (NX)*(NY)*(NZ + 1) + (NX)*(NY + 1)*(NZ) + (NX + 1)*(NY)*(NZ);
    *outStream << " Number of Elements: " << numElems << " \n";
    *outStream << "    Number of Nodes: " << numNodes << " \n";
    *outStream << "    Number of Edges: " << numEdges << " \n";
    *outStream << "    Number of Faces: " << numFaces << " \n\n";

   // Cube
    double leftX = -1.0, rightX = 1.0;
    double leftY = -1.0, rightY = 1.0;
    double leftZ = -1.0, rightZ = 1.0;

   // Mesh spacing
    double hx = (rightX-leftX)/((double)NX);
    double hy = (rightY-leftY)/((double)NY);
    double hz = (rightZ-leftZ)/((double)NZ);

   // Get nodal coordinates
    FieldContainer<double> nodeCoord(numNodes, spaceDim);
    FieldContainer<int> nodeOnBoundary(numNodes);
    int inode = 0;
    for (int k=0; k<NZ+1; k++) {
      for (int j=0; j<NY+1; j++) {
        for (int i=0; i<NX+1; i++) {
          nodeCoord(inode,0) = leftX + (double)i*hx;
          nodeCoord(inode,1) = leftY + (double)j*hy;
          nodeCoord(inode,2) = leftZ + (double)k*hz;
          if (k==0 || j==0 || i==0 || k==NZ || j==NY || i==NX){
             nodeOnBoundary(inode)=1;
          }
          inode++;
        }
      }
    }

   // Element to Node map
    FieldContainer<int> elemToNode(numElems, numNodesPerElem);
    int ielem = 0;
    for (int k=0; k<NZ; k++) {
      for (int j=0; j<NY; j++) {
        for (int i=0; i<NX; i++) {
          elemToNode(ielem,0) = (NY + 1)*(NX + 1)*k + (NX + 1)*j + i;
          elemToNode(ielem,1) = (NY + 1)*(NX + 1)*k + (NX + 1)*j + i + 1;
          elemToNode(ielem,2) = (NY + 1)*(NX + 1)*k + (NX + 1)*(j + 1) + i + 1;
          elemToNode(ielem,3) = (NY + 1)*(NX + 1)*k + (NX + 1)*(j + 1) + i;
          elemToNode(ielem,4) = (NY + 1)*(NX + 1)*(k + 1) + (NX + 1)*j + i;
          elemToNode(ielem,5) = (NY + 1)*(NX + 1)*(k + 1) + (NX + 1)*j + i + 1;
          elemToNode(ielem,6) = (NY + 1)*(NX + 1)*(k + 1) + (NX + 1)*(j + 1) + i + 1;
          elemToNode(ielem,7) = (NY + 1)*(NX + 1)*(k + 1) + (NX + 1)*(j + 1) + i;
          ielem++;
        }
      }
    }

  // Get edge connectivity
    FieldContainer<int> edgeToNode(numEdges, numNodesPerEdge);
    FieldContainer<int> elemToEdge(numElems, numEdgesPerElem);
    int iedge = 0;
    inode = 0;
    for (int k=0; k<NZ+1; k++) {
      for (int j=0; j<NY+1; j++) {
        for (int i=0; i<NX+1; i++) {
           if (i < NX){
               edgeToNode(iedge,0) = inode;
               edgeToNode(iedge,1) = inode + 1;
               if (j < NY && k < NZ){
                  ielem=i+j*NX+k*NX*NY;
                  elemToEdge(ielem,0) = iedge;
                  if (j > 0)
                     elemToEdge(ielem-NX,2) = iedge; 
                  if (k > 0)
                     elemToEdge(ielem-NX*NY,4) = iedge; 
                  if (j > 0 && k > 0)
                     elemToEdge(ielem-NX*NY-NX,6) = iedge; 
                }
               else if (j == NY && k == NZ){
                  ielem=i+(NY-1)*NX+(NZ-1)*NX*NY;
                  elemToEdge(ielem,6) = iedge;
                }
               else if (k == NZ && j < NY){
                  ielem=i+j*NX+(NZ-1)*NX*NY;
                  elemToEdge(ielem,4) = iedge;
                  if (j > 0)
                    elemToEdge(ielem-NX,6) = iedge;
                }
               else if (k < NZ && j == NY){
                  ielem=i+(NY-1)*NX+k*NX*NY;
                  elemToEdge(ielem,2) = iedge;
                  if (k > 0)
                     elemToEdge(ielem-NX*NY,6) = iedge;
                }
               iedge++;
            }
           if (j < NY){
               edgeToNode(iedge,0) = inode;
               edgeToNode(iedge,1) = inode + NX+1;
               if (i < NX && k < NZ){
                  ielem=i+j*NX+k*NX*NY;
                  elemToEdge(ielem,3) = iedge;
                  if (i > 0)
                     elemToEdge(ielem-1,1) = iedge; 
                  if (k > 0)
                     elemToEdge(ielem-NX*NY,7) = iedge; 
                  if (i > 0 && k > 0)
                     elemToEdge(ielem-NX*NY-1,5) = iedge; 
                }
               else if (i == NX && k == NZ){
                  ielem=NX-1+j*NX+(NZ-1)*NX*NY;
                  elemToEdge(ielem,5) = iedge;
                }
               else if (k == NZ && i < NX){
                  ielem=i+j*NX+(NZ-1)*NX*NY;
                  elemToEdge(ielem,7) = iedge;
                  if (i > 0)
                    elemToEdge(ielem-1,5) = iedge;
                }
               else if (k < NZ && i == NX){
                  ielem=NX-1+j*NX+k*NX*NY;
                  elemToEdge(ielem,1) = iedge;
                  if (k > 0)
                     elemToEdge(ielem-NX*NY,5) = iedge;
                }
               iedge++;
            }
           if (k < NZ){
               edgeToNode(iedge,0) = inode;
               edgeToNode(iedge,1) = inode + (NX+1)*(NY+1);
               if (i < NX && j < NY){
                  ielem=i+j*NX+k*NX*NY;
                  elemToEdge(ielem,8) = iedge;
                  if (i > 0)
                     elemToEdge(ielem-1,9) = iedge; 
                  if (j > 0)
                     elemToEdge(ielem-NX,11) = iedge; 
                  if (i > 0 && j > 0)
                     elemToEdge(ielem-NX-1,10) = iedge; 
                }
               else if (i == NX && j == NY){
                  ielem=NX-1+(NY-1)*NX+k*NX*NY;
                  elemToEdge(ielem,10) = iedge;
                }
               else if (j == NY && i < NX){
                  ielem=i+(NY-1)*NX+k*NX*NY;
                  elemToEdge(ielem,11) = iedge;
                  if (i > 0)
                    elemToEdge(ielem-1,10) = iedge;
                }
               else if (j < NY && i == NX){
                  ielem=NX-1+j*NX+k*NX*NY;
                  elemToEdge(ielem,9) = iedge;
                  if (j > 0)
                     elemToEdge(ielem-NX,10) = iedge;
                }
               iedge++;
            }
            inode++;
         }
      }
   }

   // Find boundary edges
    FieldContainer<int> edgeOnBoundary(numEdges);
    for (int i=0; i<numEdges; i++){
       if (nodeOnBoundary(edgeToNode(i,0)) && nodeOnBoundary(edgeToNode(i,1))){
           edgeOnBoundary(i)=1;
       }
    }

   // Get face connectivity
    FieldContainer<int> faceToNode(numFaces, numNodesPerFace);
    FieldContainer<int> elemToFace(numElems, numFacesPerElem);
    FieldContainer<int> faceToEdge(numFaces, numEdgesPerFace);
    int iface = 0;
    inode = 0;
    for (int k=0; k<NZ+1; k++) {
      for (int j=0; j<NY+1; j++) {
        for (int i=0; i<NX+1; i++) {
           if (i < NX && k < NZ) {
              faceToNode(iface,0)=inode;
              faceToNode(iface,1)=inode + 1;
              faceToNode(iface,2)=inode + (NX+1)*(NY+1)+1;
              faceToNode(iface,3)=inode + (NX+1)*(NY+1);
              if (j < NY) {
                 ielem=i+j*NX+k*NX*NY;
                 faceToEdge(iface,0)=elemToEdge(ielem,0);
                 faceToEdge(iface,1)=elemToEdge(ielem,9);
                 faceToEdge(iface,2)=elemToEdge(ielem,4);
                 faceToEdge(iface,3)=elemToEdge(ielem,8);
                 elemToFace(ielem,0)=iface;
                 if (j > 0) {
                    elemToFace(ielem-NX,2)=iface;
                 }
              }
              else if (j == NY) {
                 ielem=i+(NY-1)*NX+k*NX*NY;
                 faceToEdge(iface,0)=elemToEdge(ielem,2);
                 faceToEdge(iface,1)=elemToEdge(ielem,10);
                 faceToEdge(iface,2)=elemToEdge(ielem,6);
                 faceToEdge(iface,3)=elemToEdge(ielem,11);
                 elemToFace(ielem,2)=iface;
              }
              iface++;
           }
           if (j < NY && k < NZ) {
              faceToNode(iface,0)=inode;
              faceToNode(iface,1)=inode + NX+1;
              faceToNode(iface,2)=inode + (NX+1)*(NY+1) + NX+1;
              faceToNode(iface,3)=inode + (NX+1)*(NY+1);
              if (i < NX) {
                 ielem=i+j*NX+k*NX*NY;
                 faceToEdge(iface,0)=elemToEdge(ielem,3);
                 faceToEdge(iface,1)=elemToEdge(ielem,11);
                 faceToEdge(iface,2)=elemToEdge(ielem,7);
                 faceToEdge(iface,3)=elemToEdge(ielem,8);
                 elemToFace(ielem,3)=iface;
                 if (i > 0) {
                    elemToFace(ielem-1,1)=iface;
                 }
              }
              else if (i == NX) {
                 ielem=NX-1+j*NX+k*NX*NY;
                 faceToEdge(iface,0)=elemToEdge(ielem,1);
                 faceToEdge(iface,1)=elemToEdge(ielem,10);
                 faceToEdge(iface,2)=elemToEdge(ielem,5);
                 faceToEdge(iface,3)=elemToEdge(ielem,9);
                 elemToFace(ielem,1)=iface;
              }
              iface++;
           }
           if (i < NX && j < NY) {
              faceToNode(iface,0)=inode;
              faceToNode(iface,1)=inode + 1;
              faceToNode(iface,2)=inode + NX+2;
              faceToNode(iface,3)=inode + NX+1;
              if (k < NZ) {
                 ielem=i+j*NX+k*NX*NY;
                 faceToEdge(iface,0)=elemToEdge(ielem,0);
                 faceToEdge(iface,1)=elemToEdge(ielem,1);
                 faceToEdge(iface,2)=elemToEdge(ielem,2);
                 faceToEdge(iface,3)=elemToEdge(ielem,3);
                 elemToFace(ielem,4)=iface;
                 if (k > 0) {
                    elemToFace(ielem-NX*NY,5)=iface;
                 }
              }
              else if (k == NZ) {
                 ielem=i+j*NX+(NZ-1)*NX*NY;
                 faceToEdge(iface,0)=elemToEdge(ielem,4);
                 faceToEdge(iface,1)=elemToEdge(ielem,5);
                 faceToEdge(iface,2)=elemToEdge(ielem,6);
                 faceToEdge(iface,3)=elemToEdge(ielem,7);
                 elemToFace(ielem,5)=iface;
              }
              iface++;
           }
          inode++;
         }
      }
   }

   // Find boundary faces
    FieldContainer<int> faceOnBoundary(numFaces);
    for (int i=0; i<numFaces; i++){
       if (nodeOnBoundary(faceToNode(i,0)) && nodeOnBoundary(faceToNode(i,1))
          && nodeOnBoundary(faceToNode(i,2)) && nodeOnBoundary(faceToNode(i,3))){
           faceOnBoundary(i)=1;
       }
    }

#define DUMP_DATA
#ifdef DUMP_DATA
   // Output connectivity
    ofstream fe2nout("elem2node.dat");
    ofstream fe2fout("elem2face.dat");
    for (int k=0; k<NZ; k++) {
      for (int j=0; j<NY; j++) {
        for (int i=0; i<NX; i++) {
          int ielem = i + j * NX + k * NX * NY;
          for (int m=0; m<numNodesPerElem; m++){
              fe2nout << elemToNode(ielem,m) <<"  ";
           }
          fe2nout <<"\n";
          for (int n=0; n<numFacesPerElem; n++) {
             fe2fout << elemToFace(ielem,n) << "  ";
          }
          fe2fout << "\n";
        }
      }
    }
    fe2nout.close();
    fe2fout.close();
#endif

#ifdef DUMP_DATA_EXTRA
    ofstream ff2nout("face2node.dat");
    ofstream ff2eout("face2edge.dat");
    for (int i=0; i<numFaces; i++) {
       for (int j=0; j<numNodesPerFace; j++) {
           ff2nout << faceToNode(i,j) << "  ";
       }
       for (int k=0; k<numEdgesPerFace; k++) {
           ff2eout << faceToEdge(i,k) << "  ";
       }
       ff2nout << "\n";
       ff2eout << "\n";
    }
    ff2nout.close();
    ff2eout.close();

    ofstream fBnodeout("nodeOnBndy.dat");
    ofstream fBfaceout("faceOnBndy.dat");
    for (int i=0; i<numNodes; i++) {
        fBnodeout << nodeOnBoundary(i) <<"\n";
    }
    for (int i=0; i<numFaces; i++) {
        fBfaceout << faceOnBoundary(i) <<"\n";
    }
    fBnodeout.close();
    fBfaceout.close();
#endif

   // Set material properties using undeformed grid assuming each element has only one value of mu
    FieldContainer<double> muVal(numElems);
    for (int k=0; k<NZ; k++) {
      for (int j=0; j<NY; j++) {
        for (int i=0; i<NX; i++) {
          int ielem = i + j * NX + k * NX * NY;
          double midElemX = nodeCoord(elemToNode(ielem,0),0) + hx/2.0;
          double midElemY = nodeCoord(elemToNode(ielem,0),1) + hy/2.0;
          double midElemZ = nodeCoord(elemToNode(ielem,0),2) + hz/2.0;
          if ( (midElemX > mu1LeftX) && (midElemY > mu1LeftY) && (midElemZ > mu1LeftZ) &&
               (midElemX <= mu1RightX) && (midElemY <= mu1RightY) && (midElemZ <= mu1RightZ) ){
             muVal(ielem) = mu1;
          }
           else {
             muVal(ielem) = mu2;
          }
        }
      }
    }

   // Perturb mesh coordinates (only interior nodes)
    if (randomMesh){
      for (int k=1; k<NZ; k++) {
        for (int j=1; j<NY; j++) {
          for (int i=1; i<NX; i++) {
            int inode = i + j * (NX + 1) + k * (NX + 1) * (NY + 1);
           // random numbers between -1.0 and 1.0
            double rx = 2.0 * (double)rand()/RAND_MAX - 1.0;
            double ry = 2.0 * (double)rand()/RAND_MAX - 1.0;
            double rz = 2.0 * (double)rand()/RAND_MAX - 1.0; 
           // limit variation to 1/4 edge length
            nodeCoord(inode,0) = nodeCoord(inode,0) + 0.125 * hx * rx;
            nodeCoord(inode,1) = nodeCoord(inode,1) + 0.125 * hy * ry;
            nodeCoord(inode,2) = nodeCoord(inode,2) + 0.125 * hz * rz;
          }
        }
      }
    }

#ifdef DUMP_DATA
   // Print nodal coords
    ofstream fcoordout("coords.dat");
    for (int i=0; i<numNodes; i++) {
       fcoordout << nodeCoord(i,0) <<" ";
       fcoordout << nodeCoord(i,1) <<" ";
       fcoordout << nodeCoord(i,2) <<"\n";
    }
    fcoordout.close();
#endif


// **************************** INCIDENCE MATRIX **************************************

   // Edge to face incidence matrix
    *outStream << "Building incidence matrix ... \n\n";

    Epetra_SerialComm Comm;
    Epetra_Map globalMapD(numFaces, 0, Comm);
    Epetra_Map globalMapC(numEdges, 0, Comm);
    Epetra_FECrsMatrix DCurl(Copy, globalMapD, globalMapC, 2);

    double vals[4];
    vals[0]=1.0; vals[1]=1.0; vals[2]=-1.0; vals[3]=-1.0;
    for (int j=0; j<numFaces; j++){
        int rowNum = j;
        int colNum[4];
        colNum[0] = faceToEdge(j,0);
        colNum[1] = faceToEdge(j,1);
        colNum[2] = faceToEdge(j,2);
        colNum[3] = faceToEdge(j,3);
        DCurl.InsertGlobalValues(1, &rowNum, 4, colNum, vals);
    }


// ************************************ CUBATURE **************************************

   // Get numerical integration points and weights
    *outStream << "Getting cubature ... \n\n";

    DefaultCubatureFactory<double>  cubFactory;                                   
    int cubDegree = 2;
    Teuchos::RCP<Cubature<double> > hexCub = cubFactory.create(hex_8, cubDegree); 

    int cubDim       = hexCub->getDimension();
    int numCubPoints = hexCub->getNumPoints();

    FieldContainer<double> cubPoints(numCubPoints, cubDim);
    FieldContainer<double> cubWeights(numCubPoints);

    hexCub->getCubature(cubPoints, cubWeights);


   // Get numerical integration points and weights for hexahedron face
    //             (needed for rhs boundary term)

    // Define topology of the face parametrization domain as [-1,1]x[-1,1]
    CellTopology paramQuadFace(shards::getCellTopologyData<shards::Quadrilateral<4> >() );

    // Define cubature
    DefaultCubatureFactory<double>  cubFactoryFace;
    Teuchos::RCP<Cubature<double> > hexFaceCubature = cubFactoryFace.create(paramQuadFace, 3);
    int cubFaceDim    = hexFaceCubature -> getDimension();
    int numFacePoints = hexFaceCubature -> getNumPoints();

    // Define storage for cubature points and weights on [-1,1]x[-1,1]
    FieldContainer<double> paramGaussWeights(numFacePoints);
    FieldContainer<double> paramGaussPoints(numFacePoints,cubFaceDim);

    // Define storage for cubature points on workset faces
    hexFaceCubature -> getCubature(paramGaussPoints, paramGaussWeights);



// ************************************** BASIS ***************************************

   // Define basis 
    *outStream << "Getting basis ... \n\n";
    Basis_HCURL_HEX_I1_FEM<double, FieldContainer<double> > hexHCurlBasis;
    Basis_HDIV_HEX_I1_FEM<double, FieldContainer<double> > hexHDivBasis;

    int numFieldsC = hexHCurlBasis.getCardinality();
    int numFieldsD = hexHDivBasis.getCardinality();

  // Evaluate basis at cubature points
     FieldContainer<double> hexCVals(numFieldsC, numCubPoints, spaceDim); 
     FieldContainer<double> hexDVals(numFieldsD, numCubPoints, spaceDim); 
     FieldContainer<double> hexDivs(numFieldsD, numCubPoints); 
     FieldContainer<double> worksetDVals(numFieldsD, numFacePoints, spaceDim);

     hexHCurlBasis.getValues(hexCVals, cubPoints, OPERATOR_VALUE);
     hexHDivBasis.getValues(hexDVals, cubPoints, OPERATOR_VALUE);
     hexHDivBasis.getValues(hexDivs, cubPoints, OPERATOR_DIV);


// ******** LOOP OVER ELEMENTS TO CREATE LOCAL MASS and STIFFNESS MATRICES *************


    *outStream << "Building mass and stiffness matrices ... \n\n";

 // Settings and data structures for mass and stiffness matrices
    typedef CellTools<double>  CellTools;
    typedef FunctionSpaceTools fst;
    int numCells = 1; 

   // Containers for nodes, edge and face signs 
    FieldContainer<double> hexNodes(numCells, numNodesPerElem, spaceDim);
    FieldContainer<double> hexEdgeSigns(numCells, numFieldsC);
    FieldContainer<double> hexFaceSigns(numCells, numFieldsD);
   // Containers for Jacobian
    FieldContainer<double> hexJacobian(numCells, numCubPoints, spaceDim, spaceDim);
    FieldContainer<double> hexJacobInv(numCells, numCubPoints, spaceDim, spaceDim);
    FieldContainer<double> hexJacobDet(numCells, numCubPoints);
   // Containers for element HCURL mass matrix
    FieldContainer<double> massMatrixC(numCells, numFieldsC, numFieldsC);
    FieldContainer<double> weightedMeasure(numCells, numCubPoints);
    FieldContainer<double> weightedMeasureMuInv(numCells, numCubPoints);
    FieldContainer<double> hexCValsTransformed(numCells, numFieldsC, numCubPoints, spaceDim);
    FieldContainer<double> hexCValsTransformedWeighted(numCells, numFieldsC, numCubPoints, spaceDim);
   // Containers for element HDIV mass matrix
    FieldContainer<double> massMatrixD(numCells, numFieldsD, numFieldsD);
    FieldContainer<double> weightedMeasureMu(numCells, numCubPoints);    
    FieldContainer<double> hexDValsTransformed(numCells, numFieldsD, numCubPoints, spaceDim);
    FieldContainer<double> hexDValsTransformedWeighted(numCells, numFieldsD, numCubPoints, spaceDim);
   // Containers for element HDIV stiffness matrix
    FieldContainer<double> stiffMatrixD(numCells, numFieldsD, numFieldsD);
    FieldContainer<double> hexDivsTransformed(numCells, numFieldsD, numCubPoints);
    FieldContainer<double> hexDivsTransformedWeighted(numCells, numFieldsD, numCubPoints);
   // Containers for right hand side vectors
    FieldContainer<double> rhsDatag(numCells, numCubPoints, cubDim);
    FieldContainer<double> rhsDatah(numCells, numCubPoints);
    FieldContainer<double> gD(numCells, numFieldsD);
    FieldContainer<double> hD(numCells, numFieldsD);
    FieldContainer<double> gDBoundary(numCells, numFieldsD);
    FieldContainer<double> refGaussPoints(numFacePoints,spaceDim);
    FieldContainer<double> worksetGaussPoints(numCells,numFacePoints,spaceDim);
    FieldContainer<double> worksetJacobians(numCells, numFacePoints, spaceDim, spaceDim);
    FieldContainer<double> worksetJacobDet(numCells, numFacePoints);
    FieldContainer<double> worksetFaceN(numCells, numFacePoints, spaceDim);
    FieldContainer<double> worksetVFieldVals(numCells, numFacePoints, spaceDim);
    FieldContainer<double> worksetDValsTransformed(numCells, numFieldsD, numFacePoints, spaceDim);
    FieldContainer<double> curluFace(numCells, numFacePoints, spaceDim);
    FieldContainer<double> worksetDataCrossField(numCells, numFieldsD, numFacePoints, spaceDim);
   // Container for cubature points in physical space
    FieldContainer<double> physCubPoints(numCells,numCubPoints, cubDim);

    
   // Global arrays in Epetra format
    Epetra_FECrsMatrix MassC(Copy, globalMapC, numFieldsC);
    Epetra_FECrsMatrix MassD(Copy, globalMapD, numFieldsD);
    Epetra_FECrsMatrix StiffD(Copy, globalMapD, numFieldsD);
    Epetra_FEVector rhsD(globalMapD);

#ifdef DUMP_DATA
    ofstream fSignsout("faceSigns.dat");
#endif

 // *** Element loop ***
    for (int k=0; k<numElems; k++) {

     // Physical cell coordinates
      for (int i=0; i<numNodesPerElem; i++) {
         hexNodes(0,i,0) = nodeCoord(elemToNode(k,i),0);
         hexNodes(0,i,1) = nodeCoord(elemToNode(k,i),1);
         hexNodes(0,i,2) = nodeCoord(elemToNode(k,i),2);
      }

     // Face signs
      for (int j=0; j<numFacesPerElem; j++) {
         hexFaceSigns(0,j) = -1.0;
         for (int i=0; i<numNodesPerFace; i++) {
           int indf=i+1;
           if (indf > numNodesPerFace) indf=0;
           if (elemToNode(k,refFaceToNode(j,0))==faceToNode(elemToFace(k,j),i) &&
               elemToNode(k,refFaceToNode(j,1))==faceToNode(elemToFace(k,j),indf))
                hexFaceSigns(0,j) = 1.0;
          }
#ifdef DUMP_DATA
         fSignsout << hexFaceSigns(0,j) << "  ";
#endif
       }
#ifdef DUMP_DATA
       fSignsout << "\n";
#endif

     // Edge signs
      for (int j=0; j<numEdgesPerElem; j++) {
          if (elemToNode(k,refEdgeToNode(j,0))==edgeToNode(elemToEdge(k,j),0) &&
              elemToNode(k,refEdgeToNode(j,1))==edgeToNode(elemToEdge(k,j),1))
              hexEdgeSigns(0,j) = 1.0;
          else 
              hexEdgeSigns(0,j) = -1.0;
       }

    // Compute cell Jacobians, their inverses and their determinants
       CellTools::setJacobian(hexJacobian, cubPoints, hexNodes, hex_8);
       CellTools::setJacobianInv(hexJacobInv, hexJacobian );
       CellTools::setJacobianDet(hexJacobDet, hexJacobian );


// ************************** Compute element HCurl mass matrices *******************************

     // transform to physical coordinates 
      fst::HCURLtransformVALUE<double>(hexCValsTransformed, hexJacobInv, 
                                   hexCVals);
     // compute weighted measure
      fst::computeCellMeasure<double>(weightedMeasure, hexJacobDet, cubWeights);

     // multiply by weighted measure
      fst::multiplyMeasure<double>(hexCValsTransformedWeighted,
                                   weightedMeasure, hexCValsTransformed);

     // integrate to compute element mass matrix
      fst::integrate<double>(massMatrixC,
                             hexCValsTransformed, hexCValsTransformedWeighted,
                             COMP_CPP);
     // apply edge signs
      fst::applyLeftFieldSigns<double>(massMatrixC, hexEdgeSigns);
      fst::applyRightFieldSigns<double>(massMatrixC, hexEdgeSigns);

     // assemble into global matrix
      for (int row = 0; row < numFieldsC; row++){
        for (int col = 0; col < numFieldsC; col++){
            int rowIndex = elemToEdge(k,row);
            int colIndex = elemToEdge(k,col);
            double val = massMatrixC(0,row,col);
            MassC.InsertGlobalValues(1, &rowIndex, 1, &colIndex, &val);
         }
      }

// ************************** Compute element HDiv mass matrices *******************************

     // transform to physical coordinates 
      fst::HDIVtransformVALUE<double>(hexDValsTransformed, hexJacobian, hexJacobDet,
                                   hexDVals);

     // multiply by weighted measure
      fst::multiplyMeasure<double>(hexDValsTransformedWeighted,
                                   weightedMeasure, hexDValsTransformed);

     // integrate to compute element mass matrix
      fst::integrate<double>(massMatrixD,
                             hexDValsTransformed, hexDValsTransformedWeighted,
                             COMP_CPP);
     // apply face signs
      fst::applyLeftFieldSigns<double>(massMatrixD, hexFaceSigns);
      fst::applyRightFieldSigns<double>(massMatrixD, hexFaceSigns);

     // assemble into global matrix
      for (int row = 0; row < numFieldsD; row++){
        for (int col = 0; col < numFieldsD; col++){
            int rowIndex = elemToFace(k,row);
            int colIndex = elemToFace(k,col);
            double val = massMatrixD(0,row,col);
            MassD.InsertGlobalValues(1, &rowIndex, 1, &colIndex, &val);
         }
      }

// ************************ Compute element HDiv stiffness matrices *****************************

      // transform to physical coordinates 
      fst::HDIVtransformDIV<double>(hexDivsTransformed, hexJacobDet,
                                    hexDivs);

     // multiply by weighted measure
      fst::multiplyMeasure<double>(hexDivsTransformedWeighted,
                                   weightedMeasure, hexDivsTransformed);

     // integrate to compute element stiffness matrix
      fst::integrate<double>(stiffMatrixD,
                             hexDivsTransformed, hexDivsTransformedWeighted,
                             COMP_CPP);

     // apply face signs
      fst::applyLeftFieldSigns<double>(stiffMatrixD, hexFaceSigns);
      fst::applyRightFieldSigns<double>(stiffMatrixD, hexFaceSigns);

     // assemble into global matrix
      for (int row = 0; row < numFieldsD; row++){
        for (int col = 0; col < numFieldsD; col++){
            int rowIndex = elemToFace(k,row);
            int colIndex = elemToFace(k,col);
            double val = stiffMatrixD(0,row,col);
            StiffD.InsertGlobalValues(1, &rowIndex, 1, &colIndex, &val);
         }
      }

// ******************************* Build right hand side ************************************

      // transform integration points to physical points
       CellTools::mapToPhysicalFrame(physCubPoints, cubPoints, hexNodes, hex_8);

      // evaluate right hand side functions at physical points
       for (int nPt = 0; nPt < numCubPoints; nPt++){

          double x = physCubPoints(0,nPt,0);
          double y = physCubPoints(0,nPt,1);
          double z = physCubPoints(0,nPt,2);
          double du1, du2, du3;

          evalCurlCurlu(du1, du2, du3, x, y, z);
          rhsDatag(0,nPt,0) = du1;
          rhsDatag(0,nPt,1) = du2;
          rhsDatag(0,nPt,2) = du3;
         
          rhsDatah(0,nPt) = evalDivu(x, y, z);
       }

     // integrate (curl g, w) term
      fst::integrate<double>(gD, rhsDatag, hexDValsTransformedWeighted,
                             COMP_CPP);

     // integrate (h,div w) term
      fst::integrate<double>(hD, rhsDatah, hexDivsTransformedWeighted,
                             COMP_CPP);

     // apply signs
      fst::applyFieldSigns<double>(gD, hexFaceSigns);
      fst::applyFieldSigns<double>(hD, hexFaceSigns);

     // calculate boundary term, (g x w, n)_{\Gamma}
      for (int i = 0; i < numFacesPerElem; i++){
        if (faceOnBoundary(elemToFace(k,i))){

         // map Gauss points on quad to reference face: paramGaussPoints -> refGaussPoints
            CellTools::mapToReferenceSubcell(refGaussPoints,
                                   paramGaussPoints,
                                   2, i, hex_8);

         // get basis values at points on reference cell
           hexHDivBasis.getValues(worksetDVals, refGaussPoints, OPERATOR_VALUE);

         // compute Jacobians at Gauss pts. on reference face for all parent cells
           CellTools::setJacobian(worksetJacobians, refGaussPoints,
                         hexNodes, hex_8);
           CellTools::setJacobianDet(worksetJacobDet, worksetJacobians);

         // transform to physical coordinates
            fst::HDIVtransformVALUE<double>(worksetDValsTransformed, worksetJacobians,
                                   worksetJacobDet, worksetDVals);

         // map Gauss points on quad from ref. face to face workset: refGaussPoints -> worksetGaussPoints
            CellTools::mapToPhysicalFrame(worksetGaussPoints,
                                refGaussPoints,
                                hexNodes, hex_8);

        // compute face normals
            CellTools::getPhysicalFaceNormals(worksetFaceN,
                                              worksetJacobians,
                                              i, hex_8);

         // evaluate curl u at face points
           for(int nPt = 0; nPt < numFacePoints; nPt++){

             double x = worksetGaussPoints(0, nPt, 0);
             double y = worksetGaussPoints(0, nPt, 1);
             double z = worksetGaussPoints(0, nPt, 2);

             evalCurlu(curluFace(0,nPt,0), curluFace(0,nPt,1), curluFace(0,nPt,2), x, y, z);
           }

          // compute the cross product of curluFace with basis and multiply by weights
           for (int nF = 0; nF < numFieldsD; nF++){
              for(int nPt = 0; nPt < numFacePoints; nPt++){
                  worksetDataCrossField(0,nF,nPt,0) = (curluFace(0,nPt,1)*worksetDValsTransformed(0,nF,nPt,2)
                                 - curluFace(0,nPt,2)*worksetDValsTransformed(0,nF,nPt,1))
                                  * paramGaussWeights(nPt);
                  worksetDataCrossField(0,nF,nPt,1) = (curluFace(0,nPt,2)*worksetDValsTransformed(0,nF,nPt,0)
                                 - curluFace(0,nPt,0)*worksetDValsTransformed(0,nF,nPt,2))
                                  * paramGaussWeights(nPt);
                  worksetDataCrossField(0,nF,nPt,2) = (curluFace(0,nPt,0)*worksetDValsTransformed(0,nF,nPt,1)
                                 - curluFace(0,nPt,1)*worksetDValsTransformed(0,nF,nPt,0))
                                  *paramGaussWeights(nPt);
              } //nPt
           } //nF

          // Integrate
           fst::integrate<double>(gDBoundary, worksetFaceN, worksetDataCrossField,
                             COMP_CPP);

          // apply signs
           fst::applyFieldSigns<double>(gDBoundary, hexFaceSigns);

          // add into gD term
            for (int nF = 0; nF < numFieldsD; nF++){
                gD(0,nF) = gD(0,nF) - gDBoundary(0,nF);
            }

        } // if faceOnBoundary
      } // numFaces


    // assemble into global vector
     for (int row = 0; row < numFieldsD; row++){
           int rowIndex = elemToFace(k,row);
           double val = hD(0,row)+gD(0,row);
           rhsD.SumIntoGlobalValues(1, &rowIndex, &val);
     }
 
     
 } // *** end element loop ***

  // Assemble over multiple processors, if necessary
   DCurl.GlobalAssemble(); DCurl.FillComplete(MassC.RowMap(),MassD.RowMap());  
   MassC.GlobalAssemble();  MassC.FillComplete();
   MassD.GlobalAssemble();  MassD.FillComplete();
   StiffD.GlobalAssemble(); StiffD.FillComplete();
   rhsD.GlobalAssemble();
   
#ifdef DUMP_DATA
   fSignsout.close();
#endif

  // Build the inverse diagonal for MassC
   Epetra_CrsMatrix MassCinv(Copy,MassC.RowMap(),MassC.RowMap(),1);
   Epetra_Vector DiagC(MassC.RowMap());

   DiagC.PutScalar(1.0);
   MassC.Multiply(false,DiagC,DiagC);
   for(int i=0; i<DiagC.MyLength(); i++) {
     DiagC[i]=1.0/DiagC[i];
   }
   for(int i=0; i<DiagC.MyLength(); i++) {
     int CID=MassC.GCID(i);
     MassCinv.InsertGlobalValues(MassC.GRID(i),1,&(DiagC[i]),&CID);
   }
   MassCinv.FillComplete();

  // Set value to zero on diagonal that cooresponds to boundary edge
   for(int i=0;i<numEdges;i++) {
     if (edgeOnBoundary(i)){
      double val=0.0;
      MassCinv.ReplaceGlobalValues(i,1,&val,&i);
     }
   }

    int numEntries;
    double *values;
    int *cols;

  // Adjust matrices and rhs due to boundary conditions
   for (int row = 0; row<numFaces; row++){
      MassD.ExtractMyRowView(row,numEntries,values,cols);
        for (int i=0; i<numEntries; i++){
           if (faceOnBoundary(cols[i])) {
             values[i]=0;
          }
       }
      StiffD.ExtractMyRowView(row,numEntries,values,cols);
        for (int i=0; i<numEntries; i++){
           if (faceOnBoundary(cols[i])) {
             values[i]=0;
          }
       }
    }
   for (int row = 0; row<numFaces; row++){
       if (faceOnBoundary(row)) {
          int rowindex = row;
          StiffD.ExtractMyRowView(row,numEntries,values,cols);
          for (int i=0; i<numEntries; i++){
             values[i]=0;
          }
          MassD.ExtractMyRowView(row,numEntries,values,cols);
          for (int i=0; i<numEntries; i++){
             values[i]=0;
          }
         rhsD[0][row]=0;
         double val = 1.0;
         StiffD.ReplaceGlobalValues(1, &rowindex, 1, &rowindex, &val);
       }
    }

#ifdef DUMP_DATA
  // Dump matrices to disk
   EpetraExt::RowMatrixToMatlabFile("mag_m1inv_matrix.dat",MassCinv);
   EpetraExt::RowMatrixToMatlabFile("mag_m2_matrix.dat",MassD);
   EpetraExt::RowMatrixToMatlabFile("mag_k2_matrix.dat",StiffD);
   EpetraExt::RowMatrixToMatlabFile("mag_t1_matrix.dat",DCurl);
   EpetraExt::MultiVectorToMatrixMarketFile("mag_rhs2_vector.dat",rhsD,0,0,false);
#endif

   std::cout << "End Result: TEST PASSED\n";

 // reset format state of std::cout
 std::cout.copyfmt(oldFormatState);
 
 return 0;
}