Esempio n. 1
0
void box_partition_rcb( const BoxType        & root_box ,
                        std::vector<BoxType> & part_boxes )
{
  const BoxBoundsLinear use_boxes ;

  const size_t part_count = part_boxes.size();

  box_partition( 0 , part_count , root_box , & part_boxes[0] );

  // Verify partitioning

  size_t total_cell = 0 ;

  for ( size_t i = 0 ; i < part_count ; ++i ) {

    total_cell += count( part_boxes[i] );

    BoxType box_interior , box_use ;

    use_boxes.apply( root_box , part_boxes[i] , box_interior , box_use );

    if ( count( box_use ) < count( part_boxes[i] ) ||
         count( part_boxes[i] ) < count( box_interior ) ||
         part_boxes[i] != intersect( part_boxes[i] , box_use ) ||
         box_interior  != intersect( part_boxes[i] , box_interior )) {

      std::ostringstream msg ;

      msg << "box_partition_rcb ERROR : "
          << "part_boxes[" << i << "] = "
          << part_boxes[i]
          << " use " << box_use
          << " interior " << box_interior
          << std::endl 
          << "  part ^ use " << intersect( part_boxes[i] , box_use )
          << "  part ^ interior " << intersect( part_boxes[i] , box_interior );

      throw std::runtime_error( msg.str() );
    }

    for ( size_t j = i + 1 ; j < part_count ; ++j ) {
      const BoxType tmp = intersect( part_boxes[i] , part_boxes[j] );

      if ( count( tmp ) ) {
        throw std::runtime_error( std::string("box partition intersection") );
      }
    }
  }

  if ( total_cell != count( root_box ) ) {
    throw std::runtime_error( std::string("box partition count") );
  }
}
Esempio n. 2
0
  static FEMeshType create( const size_t proc_count ,
                            const size_t proc_local ,
                            const size_t gang_count ,
                            const size_t elems_x ,
                            const size_t elems_y ,
                            const size_t elems_z ,
                            const double x_coord_curve = 1 ,
                            const double y_coord_curve = 1 ,
                            const double z_coord_curve = 1 )
  {
    const size_t vertices_x = elems_x + 1 ;
    const size_t vertices_y = elems_y + 1 ;
    const size_t vertices_z = elems_z + 1 ;

    const BoxBoundsLinear vertex_box_bounds ;
    const ElementSpec element ;

    // Partition based upon vertices:

    BoxType vertex_box_global ;
    std::vector< BoxType > vertex_box_parts( proc_count );

    vertex_box_global[0][0] = 0 ; vertex_box_global[0][1] = vertices_x ;
    vertex_box_global[1][0] = 0 ; vertex_box_global[1][1] = vertices_y ;
    vertex_box_global[2][0] = 0 ; vertex_box_global[2][1] = vertices_z ;

    box_partition_rcb( vertex_box_global , vertex_box_parts );

    const BoxType vertex_box_local_owned = vertex_box_parts[ proc_local ];

    // Determine interior and used vertices:

    BoxType vertex_box_local_interior ;
    BoxType vertex_box_local_used ;

    vertex_box_bounds.apply( vertex_box_global ,
                             vertex_box_local_owned ,
                             vertex_box_local_interior ,
                             vertex_box_local_used );

    // Element counts:

    const long local_elems_x =
      ( vertex_box_local_used[0][1] - vertex_box_local_used[0][0] ) - 1 ;
    const long local_elems_y =
      ( vertex_box_local_used[1][1] - vertex_box_local_used[1][0] ) - 1 ;
    const long local_elems_z =
      ( vertex_box_local_used[2][1] - vertex_box_local_used[2][0] ) - 1 ;

    const size_t elem_count_total = std::max( long(0) , local_elems_x ) *
                                    std::max( long(0) , local_elems_y ) *
                                    std::max( long(0) , local_elems_z );

    const long interior_elems_x =
      ( vertex_box_local_owned[0][1] - vertex_box_local_owned[0][0] ) - 1 ;
    const long interior_elems_y =
      ( vertex_box_local_owned[1][1] - vertex_box_local_owned[1][0] ) - 1 ;
    const long interior_elems_z =
      ( vertex_box_local_owned[2][1] - vertex_box_local_owned[2][0] ) - 1 ;

    const size_t elem_count_interior = std::max( long(0) , interior_elems_x ) *
                                       std::max( long(0) , interior_elems_y ) *
                                       std::max( long(0) , interior_elems_z );

    // Expand vertex boxes to node boxes:

    BoxType node_box_global ;
    BoxType node_box_local_used ;
    std::vector< BoxType > node_box_parts ;

    element.create_node_boxes_from_vertex_boxes(
      vertex_box_global , vertex_box_parts ,
      node_box_global , node_box_parts );

    // Node communication maps:

    size_t node_count_interior = 0 ;
    size_t node_count_owned    = 0 ;
    size_t node_count_total    = 0 ;
    std::vector<size_t>                 node_used_id_map ;
    std::vector<size_t>                 node_part_counts ;
    std::vector< std::vector<size_t> >  node_send_map ;

    box_partition_maps( node_box_global ,
                        node_box_parts ,
                        element.box_bounds ,
                        proc_local ,
                        node_box_local_used ,
                        node_used_id_map ,
                        node_count_interior ,
                        node_count_owned ,
                        node_count_total ,
                        node_part_counts ,
                        node_send_map );

    size_t node_count_send = 0 ;
    for ( size_t i = 0 ; i < node_send_map.size() ; ++i ) {
      node_count_send += node_send_map[i].size();
    }

    size_t recv_msg_count = 0 ;
    size_t send_msg_count = 0 ;
    size_t send_count = 0 ;

    for ( size_t i = 1 ; i < proc_count ; ++i ) {
      if ( node_part_counts[i] ) ++recv_msg_count ;
      if ( node_send_map[i].size() ) {
        ++send_msg_count ;
        send_count += node_send_map[i].size();
      }
    }

    // Finite element mesh:

    FEMeshType mesh ;

    if ( node_count_total ) {
      mesh.node_coords = node_coords_type( "node_coords", node_count_total );
    }

    if ( elem_count_total ) {
      mesh.elem_node_ids =
        elem_node_ids_type( "elem_node_ids", elem_count_total );
    }

    mesh.parallel_data_map.assign( node_count_interior ,
                                   node_count_owned ,
                                   node_count_total ,
                                   recv_msg_count ,
                                   send_msg_count ,
                                   send_count );

    typename node_coords_type::HostMirror node_coords =
      Kokkos::create_mirror( mesh.node_coords );

    typename elem_node_ids_type::HostMirror elem_node_ids =
      Kokkos::create_mirror( mesh.elem_node_ids );

    //------------------------------------
    // set node coordinates to grid location for subsequent verification

    for ( size_t iz = node_box_local_used[2][0] ;
                 iz < node_box_local_used[2][1] ; ++iz ) {

    for ( size_t iy = node_box_local_used[1][0] ;
                 iy < node_box_local_used[1][1] ; ++iy ) {

    for ( size_t ix = node_box_local_used[0][0] ;
                 ix < node_box_local_used[0][1] ; ++ix ) {

      const size_t node_local_id =
        box_map_id( node_box_local_used , node_used_id_map , ix , iy , iz );

      node_coords( node_local_id , 0 ) = ix ;
      node_coords( node_local_id , 1 ) = iy ;
      node_coords( node_local_id , 2 ) = iz ;
    }}}

    //------------------------------------
    // Initialize element-node connectivity:

    if ( 1 < gang_count ) {
      layout_elements_partitioned( vertex_box_local_used ,
                                   vertex_box_local_owned ,
                                   node_box_local_used ,
                                   node_used_id_map ,
                                   element ,
                                   gang_count ,
                                   elem_node_ids );
    }
    else {
      layout_elements_interior_exterior( vertex_box_local_used ,
                                         vertex_box_local_owned ,
                                         node_box_local_used ,
                                         node_used_id_map ,
                                         element ,
                                         elem_count_interior ,
                                         elem_node_ids );
    }

    //------------------------------------
    // Populate node->element connectivity:

    std::vector<size_t> node_elem_work( node_count_total , (size_t) 0 );

    for ( size_t i = 0 ; i < elem_count_total ; ++i ) {
      for ( size_t n = 0 ; n < element_node_count  ; ++n ) {
        ++node_elem_work[ elem_node_ids(i,n) ];
      }
    }

    mesh.node_elem_ids =
      Kokkos::create_staticcrsgraph< node_elem_ids_type >( "node_elem_ids" , node_elem_work );

    typename node_elem_ids_type::HostMirror
      node_elem_ids = Kokkos::create_mirror( mesh.node_elem_ids );

    for ( size_t i = 0 ; i < node_count_total ; ++i ) {
      node_elem_work[i] = node_elem_ids.row_map[i];
    }

    // Looping in element order insures the list of elements
    // is sorted by element index.

    for ( size_t i = 0 ; i < elem_count_total ; ++i ) {
      for ( size_t n = 0 ; n < element_node_count ; ++n ) {
        const unsigned nid = elem_node_ids(i, n);
        const unsigned j = node_elem_work[nid] ; ++node_elem_work[nid] ;

        node_elem_ids.entries( j , 0 ) = i ;
        node_elem_ids.entries( j , 1 ) = n ;
      }
    }
    //------------------------------------
    // Verify setup with node coordinates matching grid indices.
    verify( node_coords , elem_node_ids , node_elem_ids );

    //------------------------------------
    // Scale node coordinates to problem extent with
    // nonlinear mapping.
    {
      const double problem_extent[3] =
        { static_cast<double>( vertex_box_global[0][1] - 1 ) ,
          static_cast<double>( vertex_box_global[1][1] - 1 ) ,
          static_cast<double>( vertex_box_global[2][1] - 1 ) };

      const double grid_extent[3] =
        { static_cast<double>( node_box_global[0][1] - 1 ) ,
          static_cast<double>( node_box_global[1][1] - 1 ) ,
          static_cast<double>( node_box_global[2][1] - 1 ) };

      for ( size_t i = 0 ; i < node_count_total ; ++i ) {
        const double x_unit = node_coords(i,0) / grid_extent[0] ;
        const double y_unit = node_coords(i,1) / grid_extent[1] ;
        const double z_unit = node_coords(i,2) / grid_extent[2] ;

        node_coords(i,0) = coordinate_scalar_type( problem_extent[0] * std::pow( x_unit , x_coord_curve ) );
        node_coords(i,1) = coordinate_scalar_type( problem_extent[1] * std::pow( y_unit , y_coord_curve ) );
        node_coords(i,2) = coordinate_scalar_type( problem_extent[2] * std::pow( z_unit , z_coord_curve ) );
      }
    }

    Kokkos::deep_copy( mesh.node_coords ,   node_coords );
    Kokkos::deep_copy( mesh.elem_node_ids , elem_node_ids );
    Kokkos::deep_copy( mesh.node_elem_ids.entries , node_elem_ids.entries );

    //------------------------------------
    // Communication lists:
    {
      recv_msg_count = 0 ;
      send_msg_count = 0 ;
      send_count = 0 ;

      for ( size_t i = 1 ; i < proc_count ; ++i ) {

        // Order sending starting with the local processor rank
        // to try to smooth out the amount of messages simultaneously
        // send to a particular processor.

        const int proc = ( proc_local + i ) % proc_count ;
        if ( node_part_counts[i] ) {
          mesh.parallel_data_map.host_recv(recv_msg_count,0) = proc ;
          mesh.parallel_data_map.host_recv(recv_msg_count,1) = node_part_counts[i] ;
          ++recv_msg_count ;
        }
        if ( node_send_map[i].size() ) {
          mesh.parallel_data_map.host_send(send_msg_count,0) = proc ;
          mesh.parallel_data_map.host_send(send_msg_count,1) = node_send_map[i].size() ;
          for ( size_t j = 0 ; j < node_send_map[i].size() ; ++j , ++send_count ) {
            mesh.parallel_data_map.host_send_item(send_count) = node_send_map[i][j] - node_count_interior ;
          }
          ++send_msg_count ;
        }
      }
    }

    return mesh ;
  }