STKUNIT_UNIT_TEST( PerformanceTestSkinning, large_cube) { stk::ParallelMachine pm = MPI_COMM_WORLD; const size_t p_size = stk::parallel_machine_size(pm); const size_t p_rank = stk::parallel_machine_rank(pm); // Every processor will be involved in detachment and skin-update up to 500 processors. // This puts 5000 elements on each process unless we are running with STL // in debug mode in which case we shrink the problem down in order // to keep things running in a reasonable amount of time. #ifdef _GLIBCXX_DEBUG const size_t NX = p_size*10, NY = 4, NZ = 5; #else const size_t NX = p_size*10, NY = 20, NZ = 25; #endif /* timings[0] = create mesh * timings[1] = intial skin mesh * timings[2] = detach mesh * timings[3] = delete skin * timings[4] = reskin mesh * timings[5] = sum(timings[0:4]) */ double timings[6] = {0}; double start_time = 0; //recreate skin for ( int test_run = 0; test_run < 4; ++test_run) { //create the mesh start_time = stk::wall_time(); stk::mesh::fixtures::HexFixture fixture(pm,NX,NY,NZ); const EntityRank element_rank = fixture.m_fem_meta.element_rank(); const EntityRank side_rank = fixture.m_fem_meta.side_rank(); stk::mesh::fem::FEMMetaData & fem_meta = fixture.m_fem_meta; stk::mesh::BulkData & mesh = fixture.m_bulk_data; stk::mesh::Part & skin_part = fem_meta.declare_part("skin_part"); fem_meta.commit(); fixture.generate_mesh(); timings[0] = stk::wall_dtime(start_time); //intial skin of the mesh start_time = stk::wall_time(); stk::mesh::skin_mesh(mesh, element_rank, &skin_part); timings[1] = stk::wall_dtime(start_time); stk::mesh::EntityVector entities_to_separate; if ( test_run < 2) { //detach 1/3 of the mesh size_t num_detached_this_proc = 0; for (size_t ix=NX/3; ix < 2*NX/3; ++ix) { for (size_t iy=0; iy < NY; ++iy) { for (size_t iz=0; iz < NZ; ++iz) { stk::mesh::Entity * element = fixture.elem(ix,iy,iz); if (element != NULL && element->owner_rank() == mesh.parallel_rank()) { entities_to_separate.push_back(element); num_detached_this_proc++; } } } } STKUNIT_EXPECT_TRUE( num_detached_this_proc > 0u ); } else { //detach middle of the mesh for (size_t ix=NX/2; ix < NX/2+1; ++ix) { for (size_t iy=NY/2; iy < NY/2+1; ++iy) { for (size_t iz=NZ/2; iz < NZ/2+1; ++iz) { stk::mesh::Entity * element = fixture.elem(ix,iy,iz); if (element != NULL && element->owner_rank() == mesh.parallel_rank()) { entities_to_separate.push_back(element); } } } } } start_time = stk::wall_time(); separate_and_skin_mesh( fem_meta, mesh, skin_part, entities_to_separate ); timings[2] = stk::wall_dtime(start_time); if (test_run%2 == 0) { // delete the skin start_time = stk::wall_time(); delete_skin( mesh, skin_part, side_rank ); timings[3] = stk::wall_dtime(start_time); //reskin the entire mesh start_time = stk::wall_time(); stk::mesh::skin_mesh( mesh, element_rank, &skin_part); timings[4] = stk::wall_dtime(start_time); } else { //update the skin timings[3] = 0; //update the skin of the mesh start_time = stk::wall_time(); update_skin( mesh, &skin_part, element_rank); timings[4] = stk::wall_dtime(start_time); } //total the timings timings[5] = 0; for (int i=0; i <5; ++i) { timings[5] += timings[i]; } stk::all_reduce(pm, stk::ReduceMax<5>(timings)); if (p_rank == 0) { std::cout << "\n\n"; switch (test_run) { case 0: std::cout << "Recreate entire skin after detaching 1/3 of the mesh:\n"; break; case 1: std::cout << "Update skin after detaching 1/3 of the mesh:\n"; break; case 2: std::cout << "Recreate entire skin after detaching middle of the mesh:\n"; break; case 3: std::cout << "Update skin after detaching middle of the mesh:\n"; break; } std::cout << "Num procs: " << p_size << "\n"; std::cout << "Mesh size: " << NX << 'x' << NY << 'x' << NZ << " = " << NX*NY*NZ << " elements\n"; std::cout << "Total time: " << timings[5] << "\n"; std::cout << "\tCreate mesh: " << timings[0] << "\n"; std::cout << "\tInitial skin: " << timings[1] << "\n"; std::cout << "\tDetach mesh: " << timings[2] << "\n"; std::cout << "\tDelete skin: " << timings[3] << "\n"; std::cout << "\tReskin: " << timings[4] << "\n"; std::cout << "\n\n"; } size_t num_skin_entities = count_skin_entities(mesh, skin_part, side_rank ); stk::all_reduce(pm, stk::ReduceSum<1>(&num_skin_entities)); size_t expected_num_skin; if ( test_run < 2) { expected_num_skin = 2*(NX*NY + NX*NZ + 3*NY*NZ); } else { expected_num_skin = 2*(NX*NY + NX*NZ + NY*NZ) + 12; } STKUNIT_EXPECT_EQUAL( num_skin_entities, expected_num_skin ); } }
bool skinning_use_case_1(stk::ParallelMachine pm) { bool passed = true; { //setup the mesh stk::mesh::fixtures::HexFixture fixture(pm,3,3,3); stk::mesh::fem::FEMMetaData & fem_meta = fixture.m_fem_meta; stk::mesh::BulkData & mesh = fixture.m_bulk_data; const stk::mesh::EntityRank element_rank = fem_meta.element_rank(); const stk::mesh::EntityRank side_rank = fem_meta.side_rank(); stk::mesh::Part & skin_part = fem_meta.declare_part("skin_part"); fem_meta.commit(); fixture.generate_mesh(); skin_mesh(mesh, element_rank, &skin_part); std::vector< stk::mesh::EntityId > elements_to_separate; //separate out the middle element elements_to_separate.push_back(fixture.elem_id(1,1,1)); separate_and_skin_mesh( fem_meta, mesh, skin_part, elements_to_separate, element_rank ); // pointer to middle_element after mesh modification. stk::mesh::Entity * middle_element = mesh.get_entity(element_rank, fixture.elem_id(1,1,1)); unsigned num_skin_entities = count_skin_entities(mesh, skin_part, side_rank); stk::all_reduce(pm, stk::ReduceSum<1>(&num_skin_entities)); //there should be 66 faces in the skin part //54 on the outside //6 on the inside attached to the entire mesh //6 on the inside attected to the element that was detached bool correct_skin = ( num_skin_entities == 66 ); bool correct_relations = true; bool correct_comm = true; //all nodes connected to the single element that has been broken off //should have relations.size() == 4 and comm.size() == 0 if (middle_element != NULL && middle_element->owner_rank() == mesh.parallel_rank()) { stk::mesh::PairIterRelation relations = middle_element->relations(NODE_RANK); for (; relations.first != relations.second; ++relations.first) { stk::mesh::Entity * current_node = (relations.first->entity()); //each node should be attached to only 1 element and 3 faces correct_relations &= ( current_node->relations().size() == 4 ); //the entire closure of the element should exist on a single process correct_comm &= ( current_node->comm().size() == 0 ); } } passed &= (correct_skin && correct_relations && correct_comm); } //seperate the entire middle layer of the mesh { //setup the mesh stk::mesh::fixtures::HexFixture fixture(pm,3,3,3); stk::mesh::fem::FEMMetaData & fem_meta = fixture.m_fem_meta; stk::mesh::BulkData & mesh = fixture.m_bulk_data; const stk::mesh::EntityRank element_rank = fem_meta.element_rank(); const stk::mesh::EntityRank side_rank = fem_meta.side_rank(); stk::mesh::Part & skin_part = fem_meta.declare_part("skin_part"); fem_meta.commit(); fixture.generate_mesh(); skin_mesh(mesh, element_rank, &skin_part); std::vector< stk::mesh::EntityId > elements_to_separate; //separate out the middle level elements_to_separate.push_back(fixture.elem_id(1,0,0)); elements_to_separate.push_back(fixture.elem_id(1,0,1)); elements_to_separate.push_back(fixture.elem_id(1,0,2)); elements_to_separate.push_back(fixture.elem_id(1,1,0)); elements_to_separate.push_back(fixture.elem_id(1,1,1)); elements_to_separate.push_back(fixture.elem_id(1,1,2)); elements_to_separate.push_back(fixture.elem_id(1,2,0)); elements_to_separate.push_back(fixture.elem_id(1,2,1)); elements_to_separate.push_back(fixture.elem_id(1,2,2)); separate_and_skin_mesh( fem_meta, mesh, skin_part, elements_to_separate, element_rank ); // pointer to middle_element after mesh modification. unsigned num_skin_entities = count_skin_entities(mesh, skin_part, side_rank); stk::all_reduce(pm, stk::ReduceSum<1>(&num_skin_entities)); //there should be 90 faces in the skin part //30 attached to each level of the mesh bool correct_skin = ( num_skin_entities == 90 ); passed &= correct_skin; } return passed; }
TEST( skinning_large_cube, skinning_large_cube) { stk::ParallelMachine pm = MPI_COMM_WORLD; const size_t p_size = stk::parallel_machine_size(pm); const size_t p_rank = stk::parallel_machine_rank(pm); // Every processor will be involved in detachment and skin-update up to 500 processors. // This puts 5000 elements on each process unless we are running with STL // in debug mode in which case we shrink the problem down in order // to keep things running in a reasonable amount of time. #ifdef _GLIBCXX_DEBUG const size_t NX = p_size*10, NY = 4, NZ = 5; #else const size_t NX = p_size*100, NY = 100, NZ = 100; #endif static const int TIMER_COUNT = 6; /* timings[0] = create mesh * timings[1] = intial skin mesh * timings[2] = detach mesh * timings[3] = delete skin * timings[4] = reskin mesh * timings[5] = sum(timings[0:4]) */ double timings[TIMER_COUNT] = {0}; double timing_sums[TIMER_COUNT] = {0}; const char* timer_names[TIMER_COUNT] = {"Create mesh", "Initial skin", "Detach mesh", "Delete skin", "Reskin", "Total time"}; double start_time = 0; size_t memory_max[2] = {0}; const char* memory_names[2] = {"Current memory", "Memory high water"}; //recreate skin for ( int test_run = 0; test_run < 4; ++test_run) { //create the mesh start_time = stk::wall_time(); stk::mesh::fixtures::HexFixture fixture(pm,NX,NY,NZ); const EntityRank element_rank = stk::topology::ELEMENT_RANK; const EntityRank side_rank = fixture.m_meta.side_rank(); stk::mesh::MetaData & fem_meta = fixture.m_meta; stk::mesh::BulkData & mesh = fixture.m_bulk_data; stk::mesh::Part & skin_part = fem_meta.declare_part("skin_part"); fem_meta.commit(); fixture.generate_mesh(); timings[0] = stk::wall_dtime(start_time); //intial skin of the mesh start_time = stk::wall_time(); { stk::mesh::PartVector add_parts(1,&skin_part); stk::mesh::skin_mesh(mesh, add_parts); } timings[1] = stk::wall_dtime(start_time); stk::mesh::EntityVector entities_to_separate; if ( test_run < 2) { //detach 1/3 of the mesh size_t num_detached_this_proc = 0; for (size_t ix=NX/3; ix < 2*NX/3; ++ix) { for (size_t iy=0; iy < NY; ++iy) { for (size_t iz=0; iz < NZ; ++iz) { stk::mesh::Entity element = fixture.elem(ix,iy,iz); if (mesh.is_valid(element) && mesh.parallel_owner_rank(element) == mesh.parallel_rank()) { entities_to_separate.push_back(element); num_detached_this_proc++; } } } } EXPECT_TRUE( num_detached_this_proc > 0u ); } else { //detach middle of the mesh for (size_t ix=NX/2; ix < NX/2+1; ++ix) { for (size_t iy=NY/2; iy < NY/2+1; ++iy) { for (size_t iz=NZ/2; iz < NZ/2+1; ++iz) { stk::mesh::Entity element = fixture.elem(ix,iy,iz); if (mesh.is_valid(element) && mesh.parallel_owner_rank(element) == mesh.parallel_rank()) { entities_to_separate.push_back(element); } } } } } start_time = stk::wall_time(); separate_and_skin_mesh( fem_meta, mesh, skin_part, entities_to_separate ); timings[2] = stk::wall_dtime(start_time); if (test_run%2 == 0) { // delete the skin start_time = stk::wall_time(); delete_skin( mesh, skin_part, side_rank ); timings[3] = stk::wall_dtime(start_time); //reskin the entire mesh start_time = stk::wall_time(); { stk::mesh::PartVector add_parts(1,&skin_part); stk::mesh::skin_mesh( mesh, add_parts); } timings[4] = stk::wall_dtime(start_time); } else { //update the skin timings[3] = 0; //update the skin of the mesh start_time = stk::wall_time(); update_skin( mesh, &skin_part, element_rank); timings[4] = stk::wall_dtime(start_time); } //total the timings timings[5] = 0; for (int i=0; i <5; ++i) { timings[5] += timings[i]; } size_t mem_now = 0, mem_hwm = 0; stk::get_memory_usage(mem_now, mem_hwm); stk::all_reduce(pm, stk::ReduceMax<5>(timings)); stk::all_reduce(pm, stk::ReduceMax<1>(&mem_now)); stk::all_reduce(pm, stk::ReduceMax<1>(&mem_hwm)); if (mem_now > memory_max[0]) { memory_max[0] = mem_now; } if (mem_hwm > memory_max[1]) { memory_max[1] = mem_hwm; } //compute sums for (int i=0; i<TIMER_COUNT; ++i) { timing_sums[i] += timings[i]; } if (p_rank == 0) { std::cout << "\n\n"; switch (test_run) { case 0: std::cout << "Recreate entire skin after detaching 1/3 of the mesh:\n"; break; case 1: std::cout << "Update skin after detaching 1/3 of the mesh:\n"; break; case 2: std::cout << "Recreate entire skin after detaching middle of the mesh:\n"; break; case 3: std::cout << "Update skin after detaching middle of the mesh:\n"; break; } std::cout << "Num procs: " << p_size << "\n"; std::cout << "Mesh size: " << NX << 'x' << NY << 'x' << NZ << " = " << NX*NY*NZ << " elements\n"; std::cout << "Total time: " << timings[5] << "\n"; std::cout << "\tCreate mesh: " << timings[0] << "\n"; std::cout << "\tInitial skin: " << timings[1] << "\n"; std::cout << "\tDetach mesh: " << timings[2] << "\n"; std::cout << "\tDelete skin: " << timings[3] << "\n"; std::cout << "\tReskin: " << timings[4] << "\n"; std::cout << "\n\n"; } size_t num_skin_entities = count_skin_entities(mesh, skin_part, side_rank ); stk::all_reduce(pm, stk::ReduceSum<1>(&num_skin_entities)); size_t expected_num_skin = 0; if ( test_run < 2) { expected_num_skin = 2*(NX*NY + NX*NZ + 3*NY*NZ); } else { expected_num_skin = 2*(NX*NY + NX*NZ + NY*NZ) + 12; } EXPECT_EQ( num_skin_entities, expected_num_skin ); } if (p_rank == 0) { stk::print_timers_and_memory(&timer_names[0], &timing_sums[0], TIMER_COUNT, &memory_names[0], &memory_max[0], 2); } stk::parallel_print_time_without_output_and_hwm(pm, timings[5]); }