static PyObject* Update_Skin(PyObject *self, PyObject *args)
{
int num;
int ret;
if(!PyArg_ParseTuple(args, "i", &num))return NULL;
ret=update_skin(&g_model[num]);
return Py_BuildValue("i", ret);
}
STKUNIT_UNIT_TEST( PerformanceTestSkinning, large_cube)
{
stk::ParallelMachine pm = MPI_COMM_WORLD;
const size_t p_size = stk::parallel_machine_size(pm);
const size_t p_rank = stk::parallel_machine_rank(pm);
// Every processor will be involved in detachment and skin-update up to 500 processors.
// This puts 5000 elements on each process unless we are running with STL
// in debug mode in which case we shrink the problem down in order
// to keep things running in a reasonable amount of time.
#ifdef _GLIBCXX_DEBUG
const size_t NX = p_size*10, NY = 4, NZ = 5;
#else
const size_t NX = p_size*10, NY = 20, NZ = 25;
#endif
/* timings[0] = create mesh
* timings[1] = intial skin mesh
* timings[2] = detach mesh
* timings[3] = delete skin
* timings[4] = reskin mesh
* timings[5] = sum(timings[0:4])
*/
double timings[6] = {0};
double start_time = 0;
//recreate skin
for ( int test_run = 0; test_run < 4; ++test_run) {
//create the mesh
start_time = stk::wall_time();
stk::mesh::fixtures::HexFixture fixture(pm,NX,NY,NZ);
const EntityRank element_rank = fixture.m_fem_meta.element_rank();
const EntityRank side_rank = fixture.m_fem_meta.side_rank();
stk::mesh::fem::FEMMetaData & fem_meta = fixture.m_fem_meta;
stk::mesh::BulkData & mesh = fixture.m_bulk_data;
stk::mesh::Part & skin_part = fem_meta.declare_part("skin_part");
fem_meta.commit();
fixture.generate_mesh();
timings[0] = stk::wall_dtime(start_time);
//intial skin of the mesh
start_time = stk::wall_time();
stk::mesh::skin_mesh(mesh, element_rank, &skin_part);
timings[1] = stk::wall_dtime(start_time);
stk::mesh::EntityVector entities_to_separate;
if ( test_run < 2) {
//detach 1/3 of the mesh
size_t num_detached_this_proc = 0;
for (size_t ix=NX/3; ix < 2*NX/3; ++ix) {
for (size_t iy=0; iy < NY; ++iy) {
for (size_t iz=0; iz < NZ; ++iz) {
stk::mesh::Entity * element = fixture.elem(ix,iy,iz);
if (element != NULL && element->owner_rank() == mesh.parallel_rank()) {
entities_to_separate.push_back(element);
num_detached_this_proc++;
}
}
}
}
STKUNIT_EXPECT_TRUE( num_detached_this_proc > 0u );
} else {
//detach middle of the mesh
for (size_t ix=NX/2; ix < NX/2+1; ++ix) {
for (size_t iy=NY/2; iy < NY/2+1; ++iy) {
for (size_t iz=NZ/2; iz < NZ/2+1; ++iz) {
stk::mesh::Entity * element = fixture.elem(ix,iy,iz);
if (element != NULL && element->owner_rank() == mesh.parallel_rank()) {
entities_to_separate.push_back(element);
}
}
}
}
}
start_time = stk::wall_time();
separate_and_skin_mesh(
fem_meta,
mesh,
skin_part,
entities_to_separate
);
timings[2] = stk::wall_dtime(start_time);
if (test_run%2 == 0) { // delete the skin
start_time = stk::wall_time();
delete_skin( mesh, skin_part, side_rank );
timings[3] = stk::wall_dtime(start_time);
//reskin the entire mesh
start_time = stk::wall_time();
stk::mesh::skin_mesh( mesh, element_rank, &skin_part);
timings[4] = stk::wall_dtime(start_time);
}
else { //update the skin
timings[3] = 0;
//update the skin of the mesh
start_time = stk::wall_time();
update_skin( mesh, &skin_part, element_rank);
timings[4] = stk::wall_dtime(start_time);
}
//total the timings
timings[5] = 0;
for (int i=0; i <5; ++i) {
timings[5] += timings[i];
}
stk::all_reduce(pm, stk::ReduceMax<5>(timings));
if (p_rank == 0) {
std::cout << "\n\n";
switch (test_run) {
case 0:
std::cout << "Recreate entire skin after detaching 1/3 of the mesh:\n";
break;
case 1:
std::cout << "Update skin after detaching 1/3 of the mesh:\n";
break;
case 2:
std::cout << "Recreate entire skin after detaching middle of the mesh:\n";
break;
case 3:
std::cout << "Update skin after detaching middle of the mesh:\n";
break;
}
std::cout << "Num procs: " << p_size << "\n";
std::cout << "Mesh size: " << NX << 'x' << NY << 'x' << NZ << " = " << NX*NY*NZ << " elements\n";
std::cout << "Total time: " << timings[5] << "\n";
std::cout << "\tCreate mesh: " << timings[0] << "\n";
std::cout << "\tInitial skin: " << timings[1] << "\n";
std::cout << "\tDetach mesh: " << timings[2] << "\n";
std::cout << "\tDelete skin: " << timings[3] << "\n";
std::cout << "\tReskin: " << timings[4] << "\n";
std::cout << "\n\n";
}
size_t num_skin_entities = count_skin_entities(mesh, skin_part, side_rank );
stk::all_reduce(pm, stk::ReduceSum<1>(&num_skin_entities));
size_t expected_num_skin;
if ( test_run < 2) {
expected_num_skin = 2*(NX*NY + NX*NZ + 3*NY*NZ);
} else {
expected_num_skin = 2*(NX*NY + NX*NZ + NY*NZ) + 12;
}
STKUNIT_EXPECT_EQUAL( num_skin_entities, expected_num_skin );
}
}
TEST( skinning_large_cube, skinning_large_cube)
{
stk::ParallelMachine pm = MPI_COMM_WORLD;
const size_t p_size = stk::parallel_machine_size(pm);
const size_t p_rank = stk::parallel_machine_rank(pm);
// Every processor will be involved in detachment and skin-update up to 500 processors.
// This puts 5000 elements on each process unless we are running with STL
// in debug mode in which case we shrink the problem down in order
// to keep things running in a reasonable amount of time.
#ifdef _GLIBCXX_DEBUG
const size_t NX = p_size*10, NY = 4, NZ = 5;
#else
const size_t NX = p_size*100, NY = 100, NZ = 100;
#endif
static const int TIMER_COUNT = 6;
/* timings[0] = create mesh
* timings[1] = intial skin mesh
* timings[2] = detach mesh
* timings[3] = delete skin
* timings[4] = reskin mesh
* timings[5] = sum(timings[0:4])
*/
double timings[TIMER_COUNT] = {0};
double timing_sums[TIMER_COUNT] = {0};
const char* timer_names[TIMER_COUNT] = {"Create mesh", "Initial skin", "Detach mesh", "Delete skin", "Reskin", "Total time"};
double start_time = 0;
size_t memory_max[2] = {0};
const char* memory_names[2] = {"Current memory", "Memory high water"};
//recreate skin
for ( int test_run = 0; test_run < 4; ++test_run) {
//create the mesh
start_time = stk::wall_time();
stk::mesh::fixtures::HexFixture fixture(pm,NX,NY,NZ);
const EntityRank element_rank = stk::topology::ELEMENT_RANK;
const EntityRank side_rank = fixture.m_meta.side_rank();
stk::mesh::MetaData & fem_meta = fixture.m_meta;
stk::mesh::BulkData & mesh = fixture.m_bulk_data;
stk::mesh::Part & skin_part = fem_meta.declare_part("skin_part");
fem_meta.commit();
fixture.generate_mesh();
timings[0] = stk::wall_dtime(start_time);
//intial skin of the mesh
start_time = stk::wall_time();
{
stk::mesh::PartVector add_parts(1,&skin_part);
stk::mesh::skin_mesh(mesh, add_parts);
}
timings[1] = stk::wall_dtime(start_time);
stk::mesh::EntityVector entities_to_separate;
if ( test_run < 2) {
//detach 1/3 of the mesh
size_t num_detached_this_proc = 0;
for (size_t ix=NX/3; ix < 2*NX/3; ++ix) {
for (size_t iy=0; iy < NY; ++iy) {
for (size_t iz=0; iz < NZ; ++iz) {
stk::mesh::Entity element = fixture.elem(ix,iy,iz);
if (mesh.is_valid(element) && mesh.parallel_owner_rank(element) == mesh.parallel_rank()) {
entities_to_separate.push_back(element);
num_detached_this_proc++;
}
}
}
}
EXPECT_TRUE( num_detached_this_proc > 0u );
} else {
//detach middle of the mesh
for (size_t ix=NX/2; ix < NX/2+1; ++ix) {
for (size_t iy=NY/2; iy < NY/2+1; ++iy) {
for (size_t iz=NZ/2; iz < NZ/2+1; ++iz) {
stk::mesh::Entity element = fixture.elem(ix,iy,iz);
if (mesh.is_valid(element) && mesh.parallel_owner_rank(element) == mesh.parallel_rank()) {
entities_to_separate.push_back(element);
}
}
}
}
}
start_time = stk::wall_time();
separate_and_skin_mesh(
fem_meta,
mesh,
skin_part,
entities_to_separate
);
timings[2] = stk::wall_dtime(start_time);
if (test_run%2 == 0) { // delete the skin
start_time = stk::wall_time();
delete_skin( mesh, skin_part, side_rank );
timings[3] = stk::wall_dtime(start_time);
//reskin the entire mesh
start_time = stk::wall_time();
{
stk::mesh::PartVector add_parts(1,&skin_part);
stk::mesh::skin_mesh( mesh, add_parts);
}
timings[4] = stk::wall_dtime(start_time);
}
else { //update the skin
timings[3] = 0;
//update the skin of the mesh
start_time = stk::wall_time();
update_skin( mesh, &skin_part, element_rank);
timings[4] = stk::wall_dtime(start_time);
}
//total the timings
timings[5] = 0;
for (int i=0; i <5; ++i) {
timings[5] += timings[i];
}
size_t mem_now = 0, mem_hwm = 0;
stk::get_memory_usage(mem_now, mem_hwm);
stk::all_reduce(pm, stk::ReduceMax<5>(timings));
stk::all_reduce(pm, stk::ReduceMax<1>(&mem_now));
stk::all_reduce(pm, stk::ReduceMax<1>(&mem_hwm));
if (mem_now > memory_max[0]) {
memory_max[0] = mem_now;
}
if (mem_hwm > memory_max[1]) {
memory_max[1] = mem_hwm;
}
//compute sums
for (int i=0; i<TIMER_COUNT; ++i) {
timing_sums[i] += timings[i];
}
if (p_rank == 0) {
std::cout << "\n\n";
switch (test_run) {
case 0:
std::cout << "Recreate entire skin after detaching 1/3 of the mesh:\n";
break;
case 1:
std::cout << "Update skin after detaching 1/3 of the mesh:\n";
break;
case 2:
std::cout << "Recreate entire skin after detaching middle of the mesh:\n";
break;
case 3:
std::cout << "Update skin after detaching middle of the mesh:\n";
break;
}
std::cout << "Num procs: " << p_size << "\n";
std::cout << "Mesh size: " << NX << 'x' << NY << 'x' << NZ << " = " << NX*NY*NZ << " elements\n";
std::cout << "Total time: " << timings[5] << "\n";
std::cout << "\tCreate mesh: " << timings[0] << "\n";
std::cout << "\tInitial skin: " << timings[1] << "\n";
std::cout << "\tDetach mesh: " << timings[2] << "\n";
std::cout << "\tDelete skin: " << timings[3] << "\n";
std::cout << "\tReskin: " << timings[4] << "\n";
std::cout << "\n\n";
}
size_t num_skin_entities = count_skin_entities(mesh, skin_part, side_rank );
stk::all_reduce(pm, stk::ReduceSum<1>(&num_skin_entities));
size_t expected_num_skin = 0;
if ( test_run < 2) {
expected_num_skin = 2*(NX*NY + NX*NZ + 3*NY*NZ);
} else {
expected_num_skin = 2*(NX*NY + NX*NZ + NY*NZ) + 12;
}
EXPECT_EQ( num_skin_entities, expected_num_skin );
}
if (p_rank == 0) {
stk::print_timers_and_memory(&timer_names[0], &timing_sums[0], TIMER_COUNT, &memory_names[0], &memory_max[0], 2);
}
stk::parallel_print_time_without_output_and_hwm(pm, timings[5]);
}
