void test_read_parallel(int num_verts)
{
Core moab;
Interface& mb = moab;
EntityHandle file_set;
ErrorCode rval;
rval = mb.create_meshset(MESHSET_SET, file_set);
CHECK_ERR(rval);
std::string opt = std::string("PARALLEL=READ_PART;PARTITION=;PARTITION_DISTRIBUTE;PARALLEL_RESOLVE_SHARED_ENTS") +
partition_method;
rval = mb.load_file(example, &file_set, opt.c_str());
CHECK_ERR(rval);
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
rval = pcomm->check_all_shared_handles();
CHECK_ERR(rval);
// get the total # owned verts
Range verts;
rval = mb.get_entities_by_type(0, MBVERTEX, verts);
CHECK_ERR(rval);
rval = pcomm->filter_pstatus(verts, PSTATUS_NOT_OWNED, PSTATUS_NOT);
CHECK_ERR(rval);
int my_num = verts.size(), total_verts;
MPI_Reduce(&my_num, &total_verts, 1, MPI_INTEGER, MPI_SUM, 0, pcomm->proc_config().proc_comm());
if (0 == pcomm->proc_config().proc_rank()) CHECK_EQUAL(total_verts, num_verts);
}
void test_read_conn()
{
Core moab;
Interface& mb = moab;
std::string opts;
get_options(opts);
ErrorCode rval = mb.load_file(conn_fname, NULL, opts.c_str());
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
#else
int procs = 1;
#endif
// Make check runs this test on one processor
if (1 == procs) {
// Get vertices
Range verts;
rval = mb.get_entities_by_type(0, MBVERTEX, verts);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)3458, verts.size());
// Get cells
Range cells;
rval = mb.get_entities_by_type(0, MBQUAD, cells);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)3456, cells.size());
}
}
void intersection_at_level(iMesh_Instance instance,
iBase_EntitySetHandle fine_set, iBase_EntitySetHandle lagr_set,
iBase_EntitySetHandle intx_set, double * dep_coords, double radius2,
int * ierr) {
*ierr = 1;
Interface * mb = MOABI;
//MPI_Comm mpicomm = MPI_Comm_f2c(comm);
// instantiate parallel comm now or not?
EntityHandle lagrMeshSet = (EntityHandle) lagr_set;
ParallelComm *pcomm = ParallelComm::get_pcomm(mb, 0);
if (NULL == pcomm)
return; // error is 1
// set the departure tag on the fine mesh vertices
ErrorCode rval = set_departure_points_position(mb, lagrMeshSet, dep_coords,
radius2);
ERRORV(rval, "can't set departure tag");
if (debug) {
std::stringstream fff;
fff << "lagr0" << pcomm->proc_config().proc_rank() << ".vtk";
rval = mb->write_mesh(fff.str().c_str(), &lagrMeshSet, 1);
ERRORV(rval, "can't write covering set ");
}
// it should be done earlier
pworker->SetRadius(radius);
EntityHandle covering_set;
rval = pworker->create_departure_mesh_3rd_alg(lagrMeshSet, covering_set);
ERRORV(rval, "can't compute covering set ");
if (debug) {
std::stringstream fff;
fff << "cover" << pcomm->proc_config().proc_rank() << ".vtk";
rval = mb->write_mesh(fff.str().c_str(), &covering_set, 1);
ERRORV(rval, "can't write covering set ");
}
EntityHandle intxSet = (EntityHandle) intx_set;
rval = pworker->intersect_meshes(covering_set, (EntityHandle) fine_set,
intxSet);
ERRORV(rval, "can't intersect ");
if (debug) {
std::stringstream fff;
fff << "intx0" << pcomm->proc_config().proc_rank() << ".vtk";
rval = mb->write_mesh(fff.str().c_str(), &intxSet, 1);
ERRORV(rval, "can't write covering set ");
}
return;
}
void test_read_onevar()
{
Core moab;
Interface& mb = moab;
std::string opts;
get_options(opts);
// Read mesh and read vertex variable T at all timesteps
opts += std::string(";VARIABLE=T");
ErrorCode rval = mb.load_file(example, NULL, opts.c_str());
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
#else
int procs = 1;
#endif
// Make check runs this test on one processor
if (1 == procs) {
// Check for proper tags
Tag Ttag0, Ttag1;
rval = mb.tag_get_handle("T0", levels, MB_TYPE_DOUBLE, Ttag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("T1", levels, MB_TYPE_DOUBLE, Ttag1);
CHECK_ERR(rval);
// Get vertices
Range verts;
rval = mb.get_entities_by_type(0, MBVERTEX, verts);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)3458, verts.size());
// Get all values of tag T0
int count;
void* Tbuf;
rval = mb.tag_iterate(Ttag0, verts.begin(), verts.end(), count, Tbuf);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)count, verts.size());
const double eps = 0.0001;
double* data = (double*) Tbuf;
// Check first level values on 4 strategically selected vertices
CHECK_REAL_EQUAL(233.1136, data[0 * levels], eps); // First vert
CHECK_REAL_EQUAL(236.1505, data[1728 * levels], eps); // Median vert
CHECK_REAL_EQUAL(235.7722, data[1729 * levels], eps); // Median vert
CHECK_REAL_EQUAL(234.0416, data[3457 * levels], eps); // Last vert
}
}
ErrorCode test_read(const char *filename, const char *option)
{
Core mb_instance;
Interface& moab = mb_instance;
ErrorCode rval;
rval = moab.load_file( filename, 0, option);
CHKERR(rval);
ParallelComm* pcomm = ParallelComm::get_pcomm(&moab, 0);
rval = pcomm->check_all_shared_handles();
CHKERR(rval);
return MB_SUCCESS;
}
/**
Perform full parallel sweep algorithm on subset of subdomains.
*/
void SweepSubdomains (std::vector<int> subdomain_list, Grid_Data *grid_data, bool block_jacobi)
{
// Create a new sweep communicator object
ParallelComm *comm = NULL;
if(block_jacobi){
comm = new BlockJacobiComm(grid_data);
}
else {
comm = new SweepComm(grid_data);
}
// Add all subdomains in our list
for(int i = 0;i < subdomain_list.size();++ i){
int sdom_id = subdomain_list[i];
comm->addSubdomain(sdom_id, grid_data->subdomains[sdom_id]);
}
/* Loop until we have finished all of our work */
while(comm->workRemaining()){
// Get a list of subdomains that have met dependencies
std::vector<int> sdom_ready = comm->readySubdomains();
int backlog = sdom_ready.size();
// Run top of list
if(backlog > 0){
int sdom_id = sdom_ready[0];
Subdomain &sdom = grid_data->subdomains[sdom_id];
// Clear boundary conditions
for(int dim = 0;dim < 3;++ dim){
if(sdom.upwind[dim].subdomain_id == -1){
sdom.plane_data[dim]->clear(0.0);
}
}
{
BLOCK_TIMER(grid_data->timing, Sweep_Kernel);
// Perform subdomain sweep
grid_data->kernel->sweep(&sdom);
}
// Mark as complete (and do any communication)
comm->markComplete(sdom_id);
}
}
delete comm;
}
void multiple_loads_of_same_file()
{
Core moab;
Interface& mb = moab;
// Need a file set for nomesh to work right
EntityHandle file_set;
ErrorCode rval;
rval = mb.create_meshset(MESHSET_SET, file_set);
CHECK_ERR(rval);
// Read first only header information, no mesh, no variable
read_options = "PARALLEL=READ_PART;PARTITION;NOMESH;VARIABLE=;PARTITION_METHOD=TRIVIAL";
rval = mb.load_file(example, &file_set, read_options.c_str());
CHECK_ERR(rval);
// Create mesh, no variable
read_options = "PARALLEL=READ_PART;PARTITION;PARALLEL_RESOLVE_SHARED_ENTS;PARTITION_METHOD=TRIVIAL;VARIABLE=";
rval = mb.load_file(example, &file_set, read_options.c_str());
CHECK_ERR(rval);
// Read variable vorticity at timestep 0, no mesh
read_options = "PARALLEL=READ_PART;PARTITION;PARTITION_METHOD=TRIVIAL;NOMESH;VARIABLE=vorticity;TIMESTEP=0";
rval = mb.load_file(example, &file_set, read_options.c_str());
CHECK_ERR(rval);
Range local_verts;
rval = mb.get_entities_by_type(file_set, MBVERTEX, local_verts);
CHECK_ERR(rval);
Range local_edges;
rval = mb.get_entities_by_type(file_set, MBEDGE, local_edges);
CHECK_ERR(rval);
Range local_cells;
rval = mb.get_entities_by_type(file_set, MBPOLYGON, local_cells);
CHECK_ERR(rval);
// No mixed elements
CHECK_EQUAL((size_t)1, local_cells.psize());
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
int rank = pcomm->proc_config().proc_rank();
// Make check runs this test on two processors
if (2 == procs) {
CHECK_EQUAL((size_t)321, local_cells.size());
// Check tag for cell variable vorticity at timestep 0
Tag vorticity_tag0;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0);
CHECK_ERR(rval);
// Get vorticity0 tag values on 3 local cells
double vorticity0_val[3 * layers];
EntityHandle cell_ents[] = {local_cells[0], local_cells[160], local_cells[320]};
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 3, vorticity0_val);
CHECK_ERR(rval);
if (0 == rank) {
CHECK_EQUAL((size_t)687, local_verts.size());
CHECK_EQUAL((size_t)1007, local_edges.size());
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.708188, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(0.131688, vorticity0_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.708164, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(0.131686, vorticity0_val[2 * layers + 1], eps);
}
else if (1 == rank) {
CHECK_EQUAL((size_t)688, local_verts.size());
CHECK_EQUAL((size_t)1008, local_edges.size());
// Layer 0
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(2.434397, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(2.434363, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps);
}
}
}
void read_mesh_parallel(bool rcbzoltan)
{
Core moab;
Interface& mb = moab;
read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=";
if (rcbzoltan)
read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=";
ErrorCode rval = mb.load_file(example, NULL, read_options.c_str());
CHECK_ERR(rval);
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
int rank = pcomm->proc_config().proc_rank();
rval = pcomm->check_all_shared_handles();
CHECK_ERR(rval);
// Get local vertices
Range local_verts;
rval = mb.get_entities_by_type(0, MBVERTEX, local_verts);
CHECK_ERR(rval);
int verts_num = local_verts.size();
if (2 == procs) {
if (rcbzoltan) {
if (0 == rank)
CHECK_EQUAL(684, verts_num);
else if (1 == rank)
CHECK_EQUAL(691, verts_num); // Not owned vertices included
}
else {
if (0 == rank)
CHECK_EQUAL(687, verts_num);
else if (1 == rank)
CHECK_EQUAL(688, verts_num); // Not owned vertices included
}
}
rval = pcomm->filter_pstatus(local_verts, PSTATUS_NOT_OWNED, PSTATUS_NOT);
CHECK_ERR(rval);
verts_num = local_verts.size();
if (2 == procs) {
if (rcbzoltan) {
if (0 == rank)
CHECK_EQUAL(684, verts_num);
else if (1 == rank)
CHECK_EQUAL(596, verts_num); // Not owned vertices excluded
}
else {
if (0 == rank)
CHECK_EQUAL(687, verts_num);
else if (1 == rank)
CHECK_EQUAL(593, verts_num); // Not owned vertices excluded
}
}
// Get local edges
Range local_edges;
rval = mb.get_entities_by_type(0, MBEDGE, local_edges);
CHECK_ERR(rval);
int edges_num = local_edges.size();
if (2 == procs) {
if (rcbzoltan) {
if (0 == rank)
CHECK_EQUAL(1002, edges_num);
else if (1 == rank)
CHECK_EQUAL(1013, edges_num); // Not owned edges included
}
else {
if (0 == rank)
CHECK_EQUAL(1007, edges_num);
else if (1 == rank)
CHECK_EQUAL(1008, edges_num); // Not owned edges included
}
}
rval = pcomm->filter_pstatus(local_edges, PSTATUS_NOT_OWNED, PSTATUS_NOT);
CHECK_ERR(rval);
edges_num = local_edges.size();
if (2 == procs) {
if (rcbzoltan) {
if (0 == rank)
CHECK_EQUAL(1002, edges_num);
else if (1 == rank)
CHECK_EQUAL(918, edges_num); // Not owned edges excluded
}
else {
if (0 == rank)
CHECK_EQUAL(1007, edges_num);
else if (1 == rank)
CHECK_EQUAL(913, edges_num); // Not owned edges excluded
}
}
// Get local cells
Range local_cells;
rval = mb.get_entities_by_type(0, MBPOLYGON, local_cells);
CHECK_ERR(rval);
// No mixed elements
CHECK_EQUAL((size_t)1, local_cells.psize());
int cells_num = local_cells.size();
if (2 == procs) {
if (rcbzoltan) {
if (0 == rank)
CHECK_EQUAL(319, cells_num);
else
CHECK_EQUAL(323, cells_num);
}
else
CHECK_EQUAL(321, cells_num);
}
rval = pcomm->filter_pstatus(local_cells, PSTATUS_NOT_OWNED, PSTATUS_NOT);
CHECK_ERR(rval);
cells_num = local_cells.size();
if (2 == procs) {
if (rcbzoltan) {
if (0 == rank)
CHECK_EQUAL(319, cells_num);
else
CHECK_EQUAL(323, cells_num);
}
else
CHECK_EQUAL(321, cells_num);
}
std::cout << "proc: " << rank << " verts:" << verts_num << "\n";
int total_verts_num;
MPI_Reduce(&verts_num, &total_verts_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm());
if (0 == rank) {
std::cout << "total vertices: " << total_verts_num << "\n";
CHECK_EQUAL(1280, total_verts_num);
}
std::cout << "proc: " << rank << " edges:" << edges_num << "\n";
int total_edges_num;
MPI_Reduce(&edges_num, &total_edges_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm());
if (0 == rank) {
std::cout << "total edges: " << total_edges_num << "\n";
CHECK_EQUAL(1920, total_edges_num);
}
std::cout << "proc: " << rank << " cells:" << cells_num << "\n";
int total_cells_num;
MPI_Reduce(&cells_num, &total_cells_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm());
if (0 == rank) {
std::cout << "total cells: " << total_cells_num << "\n";
CHECK_EQUAL(642, total_cells_num);
}
#ifdef MOAB_HAVE_HDF5_PARALLEL
std::string write_options("PARALLEL=WRITE_PART;");
std::string output_file = "test_gcrm";
if (rcbzoltan)
output_file += "_rcbzoltan";
output_file += ".h5m";
mb.write_file(output_file.c_str(), NULL, write_options.c_str());
#endif
}
void gather_one_cell_var(int gather_set_rank)
{
Core moab;
Interface& mb = moab;
EntityHandle file_set;
ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set);
CHECK_ERR(rval);
read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS";
std::ostringstream gather_set_option;
gather_set_option << ";GATHER_SET=" << gather_set_rank;
read_options += gather_set_option.str();
rval = mb.load_file(example, &file_set, read_options.c_str());
CHECK_ERR(rval);
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
int rank = pcomm->proc_config().proc_rank();
// Make sure gather_set_rank is valid
if (gather_set_rank < 0 || gather_set_rank >= procs)
return;
Range cells, cells_owned;
rval = mb.get_entities_by_type(file_set, MBPOLYGON, cells);
CHECK_ERR(rval);
// Get local owned cells
rval = pcomm->filter_pstatus(cells, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &cells_owned);
CHECK_ERR(rval);
EntityHandle gather_set = 0;
if (gather_set_rank == rank) {
// Get gather set
ReadUtilIface* readUtilIface;
mb.query_interface(readUtilIface);
rval = readUtilIface->get_gather_set(gather_set);
CHECK_ERR(rval);
assert(gather_set != 0);
}
Tag vorticity_tag0, gid_tag;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0, MB_TAG_DENSE);
CHECK_ERR(rval);
rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE);
CHECK_ERR(rval);
pcomm->gather_data(cells_owned, vorticity_tag0, gid_tag, gather_set, gather_set_rank);
if (gather_set_rank == rank) {
// Get gather set cells
Range gather_set_cells;
rval = mb.get_entities_by_type(gather_set, MBPOLYGON, gather_set_cells);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)642, gather_set_cells.size());
CHECK_EQUAL((size_t)1, gather_set_cells.psize());
// Check vorticity0 tag values on 4 gather set cells: first cell, two median cells, and last cell
EntityHandle cell_ents[] = {gather_set_cells[0], gather_set_cells[320],
gather_set_cells[321], gather_set_cells[641]};
double vorticity0_val[4 * layers];
rval = mb.tag_get_data(vorticity_tag0, &cell_ents[0], 4, vorticity0_val);
CHECK_ERR(rval);
// Only check first two layers
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(0.131688, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[3 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(0.131686, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[3 * layers + 1], eps);
}
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
LONG_DESC << "This program simulates a transport problem on a sphere"
" according to a benchmark from a Nair & Lauritzen paper.\n"
<< "It starts with a partitioned mesh on a sphere, add a tracer, and steps through.\n" <<
"The flow reverses after half time, and it should return to original configuration, if the integration was exact. ";
ProgOptions opts(LONG_DESC.str(), BRIEF_DESC);
// read a homme file, partitioned in 16 so far
std::string fileN= TestDir + "/HN16.h5m";
const char *filename_mesh1 = fileN.c_str();
opts.addOpt<double>("gtolerance,g",
"geometric absolute tolerance (used for point concidence on the sphere)", >ol);
std::string input_file;
opts.addOpt<std::string>("input_file,i", "input mesh file, partitioned",
&input_file);
std::string extra_read_opts;
opts.addOpt<std::string>("extra_read_options,O", "extra read options ",
&extra_read_opts);
//int field_type;
opts.addOpt<int>("field_type,f",
"field type-- 1: quasi-smooth; 2: smooth; 3: slotted cylinders (non-smooth)", &field_type);
opts.addOpt<int>("num_steps,n",
"number of steps ", &numSteps);
//bool reorder = false;
opts.addOpt<void>("write_debug_files,w", "write debugging files during simulation ",
&writeFiles);
opts.addOpt<void>("write_velocity_files,v", "Reorder mesh to group entities by partition",
&velocity);
opts.addOpt<void>("write_result_in_parallel,p", "write tracer result files",
¶llelWrite);
opts.parseCommandLine(argc, argv);
if (!input_file.empty())
filename_mesh1=input_file.c_str();
// read in parallel, in the "euler_set", the initial mesh
std::string optsRead = std::string("PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION")+
std::string(";PARALLEL_RESOLVE_SHARED_ENTS")+extra_read_opts;
Core moab;
Interface & mb = moab;
EntityHandle euler_set;
ErrorCode rval;
rval = mb.create_meshset(MESHSET_SET, euler_set);
CHECK_ERR(rval);
rval = mb.load_file(filename_mesh1, &euler_set, optsRead.c_str());
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
CHECK_ERR(rval);
rval = pcomm->check_all_shared_handles();
CHECK_ERR(rval);
int rank = pcomm->proc_config().proc_rank();
if (0==rank)
{
std::cout << " case 1: use -gtol " << gtol <<
" -R " << radius << " -input " << filename_mesh1 << " -f " << field_type <<
" numSteps: " << numSteps << "\n";
std::cout<<" write debug results: " << (writeFiles ? "yes" : "no") << "\n";
std::cout<< " write tracer in parallel: " << ( parallelWrite ? "yes" : "no") << "\n";
std::cout <<" output velocity: " << (velocity? "yes" : "no") << "\n";
}
// tagTracer is the value at nodes
Tag tagTracer = 0;
std::string tag_name("Tracer");
rval = mb.tag_get_handle(tag_name.c_str(), 1, MB_TYPE_DOUBLE, tagTracer, MB_TAG_DENSE | MB_TAG_CREAT);
CHECK_ERR(rval);
// tagElem is the average computed at each element, from nodal values
Tag tagElem = 0;
std::string tag_name2("TracerAverage");
rval = mb.tag_get_handle(tag_name2.c_str(), 1, MB_TYPE_DOUBLE, tagElem, MB_TAG_DENSE | MB_TAG_CREAT);
CHECK_ERR(rval);
// area of the euler element is fixed, store it; it is used to recompute the averages at each
// time step
Tag tagArea = 0;
std::string tag_name4("Area");
rval = mb.tag_get_handle(tag_name4.c_str(), 1, MB_TYPE_DOUBLE, tagArea, MB_TAG_DENSE | MB_TAG_CREAT);
CHECK_ERR(rval);
// add a field value, quasi smooth first
rval = add_field_value(&mb, euler_set, rank, tagTracer, tagElem, tagArea);
CHECK_ERR(rval);
// iniVals are used for 1-norm error computation
Range redEls;
rval = mb.get_entities_by_dimension(euler_set, 2, redEls);
CHECK_ERR(rval);
std::vector<double> iniVals(redEls.size());
rval = mb.tag_get_data(tagElem, redEls, &iniVals[0]);
CHECK_ERR(rval);
Tag tagh = 0;
std::string tag_name3("Case1");
rval = mb.tag_get_handle(tag_name3.c_str(), 3, MB_TYPE_DOUBLE, tagh, MB_TAG_DENSE | MB_TAG_CREAT);
CHECK_ERR(rval);
EntityHandle out_set, lagr_set;
rval = mb.create_meshset(MESHSET_SET, out_set);
CHECK_ERR(rval);
rval = mb.create_meshset(MESHSET_SET, lagr_set);
CHECK_ERR(rval);
// copy the initial mesh in the lagrangian set
// initial vertices will be at the same position as euler;
rval = create_lagr_mesh(&mb, euler_set, lagr_set);
CHECK_ERR(rval);
Intx2MeshOnSphere worker(&mb);
worker.SetRadius(radius);
worker.SetErrorTolerance(gtol);
Range local_verts;
rval = worker.build_processor_euler_boxes(euler_set, local_verts);// output also the local_verts
// these stay fixed for one run
// other things from intersection might need to change, like input blue set (departure set)
// so we need also a method to clean memory
CHECK_ERR(rval);
for (int i=1; i<numSteps+1; i++)
{
// time depends on i; t = i*T/numSteps: ( 0, T/numSteps, 2*T/numSteps, ..., T )
// this is really just to create some plots; it is not really needed to proceed
// the compute_tracer_case1 method actually computes the departure point position
if (velocity)
{
rval = compute_velocity_case1(&mb, euler_set, tagh, rank, i);
CHECK_ERR(rval);
}
// this is to actually compute concentrations at time step i, using the
// current concentrations
//
rval = compute_tracer_case1(&mb, worker, euler_set, lagr_set, out_set,
tagElem, tagArea, rank, i, local_verts);
CHECK_ERR(rval);
}
//final vals and 1-norm
Range::iterator iter = redEls.begin();
double norm1 = 0.;
int count =0;
void * data;
int j=0;// index in iniVals
while (iter != redEls.end())
{
rval = mb.tag_iterate(tagElem, iter, redEls.end(), count, data);
CHECK_ERR(rval);
double * ptrTracer=(double*)data;
rval = mb.tag_iterate(tagArea, iter, redEls.end(), count, data);
CHECK_ERR(rval);
double * ptrArea=(double*)data;
for (int i=0; i<count; i++, iter++, ptrTracer++, ptrArea++, j++)
{
//double area = *ptrArea;
norm1+=fabs(*ptrTracer - iniVals[j])* (*ptrArea);
}
}
double total_norm1=0;
int mpi_err = MPI_Reduce(&norm1, &total_norm1, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
if (MPI_SUCCESS != mpi_err) return 1;
if (0==rank)
std::cout << " numSteps:" << numSteps << " 1-norm:" << total_norm1 << "\n";
MPI_Finalize();
return 0;
}
// Helper functions
void read_one_cell_var(bool rcbzoltan)
{
Core moab;
Interface& mb = moab;
read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;NO_EDGES;VARIABLE=vorticity";
if (rcbzoltan)
read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;NO_EDGES;VARIABLE=vorticity;DEBUG_IO=1";
ErrorCode rval = mb.load_file(example, NULL, read_options.c_str());
CHECK_ERR(rval);
// Get local edges
Range local_edges;
rval = mb.get_entities_by_type(0, MBEDGE, local_edges);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)0, local_edges.size());
// Get local cells
Range local_cells;
rval = mb.get_entities_by_type(0, MBPOLYGON, local_cells);
CHECK_ERR(rval);
// No mixed elements
CHECK_EQUAL((size_t)1, local_cells.psize());
Tag gid_tag;
rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE);
CHECK_ERR(rval);
std::vector<int> gids(local_cells.size());
rval = mb.tag_get_data(gid_tag, local_cells, &gids[0]);
Range local_cell_gids;
std::copy(gids.rbegin(), gids.rend(), range_inserter(local_cell_gids));
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
int rank = pcomm->proc_config().proc_rank();
// Make check runs this test on two processors
if (2 == procs) {
// Check tag for cell variable vorticity at timestep 0
Tag vorticity_tag0;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0);
CHECK_ERR(rval);
// Check tag for cell variable vorticity at timestep 1
Tag vorticity_tag1;
rval = mb.tag_get_handle("vorticity1", layers, MB_TYPE_DOUBLE, vorticity_tag1);
CHECK_ERR(rval);
// Get vorticity0 and vorticity1 tag values on 3 local cells
double vorticity0_val[3 * layers];
double vorticity1_val[3 * layers];
if (rcbzoltan) {
CHECK_EQUAL((size_t)14, local_cell_gids.psize());
if (0 == rank) {
CHECK_EQUAL((size_t)319, local_cells.size());
CHECK_EQUAL((size_t)319, local_cell_gids.size());
CHECK_EQUAL(3, (int)local_cell_gids[0]);
CHECK_EQUAL(162, (int)local_cell_gids[159]);
CHECK_EQUAL(642, (int)local_cell_gids[318]);
EntityHandle cell_ents[] = {local_cells[0], local_cells[159], local_cells[318]};
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 3, vorticity0_val);
CHECK_ERR(rval);
// Timestep 0
// Layer 0
CHECK_REAL_EQUAL(-0.725999, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.814997, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-0.725989, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.814972, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps);
rval = mb.tag_get_data(vorticity_tag1, cell_ents, 3, vorticity1_val);
CHECK_ERR(rval);
// Timestep 1
// Layer 0
CHECK_REAL_EQUAL(-0.706871, vorticity1_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.767178, vorticity1_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, vorticity1_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-0.706861, vorticity1_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.767153, vorticity1_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, vorticity1_val[2 * layers + 1], eps);
}
else if (1 == rank) {
CHECK_EQUAL((size_t)323, local_cells.size());
CHECK_EQUAL((size_t)323, local_cell_gids.size());
CHECK_EQUAL(1, (int)local_cell_gids[0]);
CHECK_EQUAL(365, (int)local_cell_gids[161]);
CHECK_EQUAL(557, (int)local_cell_gids[322]);
EntityHandle cell_ents[] = {local_cells[0], local_cells[161], local_cells[322]};
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 3, vorticity0_val);
CHECK_ERR(rval);
// Timestep 0
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.173971, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(3.526371, vorticity0_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.173955, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(3.526322, vorticity0_val[2 * layers + 1], eps);
rval = mb.tag_get_data(vorticity_tag1, cell_ents, 3, vorticity1_val);
CHECK_ERR(rval);
// Timestep 1
// Layer 0
CHECK_REAL_EQUAL(3.534355, vorticity1_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.143041, vorticity1_val[1 * layers], eps);
CHECK_REAL_EQUAL(3.433463, vorticity1_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.534306, vorticity1_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.143025, vorticity1_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(3.433415, vorticity1_val[2 * layers + 1], eps);
}
}
else {
CHECK_EQUAL((size_t)321, local_cells.size());
CHECK_EQUAL((size_t)321, local_cell_gids.size());
CHECK_EQUAL((size_t)1, local_cell_gids.psize());
EntityHandle cell_ents[] = {local_cells[0], local_cells[160], local_cells[320]};
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 3, vorticity0_val);
CHECK_ERR(rval);
rval = mb.tag_get_data(vorticity_tag1, cell_ents, 3, vorticity1_val);
CHECK_ERR(rval);
if (0 == rank) {
CHECK_EQUAL(1, (int)local_cell_gids[0]);
CHECK_EQUAL(161, (int)local_cell_gids[160]);
CHECK_EQUAL(321, (int)local_cell_gids[320]);
// Timestep 0
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.708188, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(0.131688, vorticity0_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.708164, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(0.131686, vorticity0_val[2 * layers + 1], eps);
// Timestep 1
// Layer 0
CHECK_REAL_EQUAL(3.534355, vorticity1_val[0 * layers], eps);
CHECK_REAL_EQUAL(-1.663182, vorticity1_val[1 * layers], eps);
CHECK_REAL_EQUAL(0.128218, vorticity1_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.534306, vorticity1_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-1.663160, vorticity1_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(0.128216, vorticity1_val[2 * layers + 1], eps);
}
else if (1 == rank) {
CHECK_EQUAL(322, (int)local_cell_gids[0]);
CHECK_EQUAL(482, (int)local_cell_gids[160]);
CHECK_EQUAL(642, (int)local_cell_gids[320]);
// Timestep 0
// Layer 0
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(2.434397, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(2.434363, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps);
// Timestep 1
// Layer 0
CHECK_REAL_EQUAL(-0.540269, vorticity1_val[0 * layers], eps);
CHECK_REAL_EQUAL(2.370258, vorticity1_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, vorticity1_val[2 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-0.540262, vorticity1_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(2.370226, vorticity1_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, vorticity1_val[2 * layers + 1], eps);
}
}
}
}
void test_read_eul_onevar()
{
Core moab;
Interface& mb = moab;
std::string opts;
ErrorCode rval = get_options(opts);
CHECK_ERR(rval);
opts += std::string(";VARIABLE=T");
rval = mb.load_file(example_eul, NULL, opts.c_str());
CHECK_ERR(rval);
// Check for proper tags
Tag Ttag0, Ttag1;
rval = mb.tag_get_handle("T0", levels, MB_TYPE_DOUBLE, Ttag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("T1", levels, MB_TYPE_DOUBLE, Ttag1);
CHECK_ERR(rval);
// Check values of tag T0 (first level) at some strategically chosen places below
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int rank = pcomm->proc_config().proc_rank();
int procs = pcomm->proc_config().proc_size();
#else
int rank = 0;
int procs = 1;
#endif
const double eps = 0.0001;
double val[8 * levels];
if (1 == procs) {
Range global_quads;
rval = mb.get_entities_by_type(0, MBQUAD, global_quads);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)4608, global_quads.size());
EntityHandle gloabl_quad_ents[] = {global_quads[0], global_quads[2255], global_quads[2304], global_quads[4559],
global_quads[48], global_quads[2303], global_quads[2352], global_quads[4607]};
rval = mb.tag_get_data(Ttag0, &gloabl_quad_ents[0], 8, val);
CHECK_REAL_EQUAL(252.8529, val[0 * levels], eps); // First global quad
CHECK_REAL_EQUAL(234.8390, val[1 * levels], eps); // 2256th global quad
CHECK_REAL_EQUAL(232.6458, val[2 * levels], eps); // 2305th global quad
CHECK_REAL_EQUAL(205.3905, val[3 * levels], eps); // 4560th global quad
CHECK_REAL_EQUAL(252.7116, val[4 * levels], eps); // 49th global quad
CHECK_REAL_EQUAL(232.6670, val[5 * levels], eps); // 2304th global quad
CHECK_REAL_EQUAL(234.6922, val[6 * levels], eps); // 2353th global quad
CHECK_REAL_EQUAL(200.6828, val[7 * levels], eps); // Last global quad
}
else if (2 == procs) {
Range local_quads;
rval = mb.get_entities_by_type(0, MBQUAD, local_quads);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)2304, local_quads.size());
EntityHandle local_quad_ents[] = {local_quads[0], local_quads[1151], local_quads[1152], local_quads[2303]};
rval = mb.tag_get_data(Ttag0, &local_quad_ents[0], 4, val);
if (0 == rank) {
CHECK_REAL_EQUAL(252.8529, val[0 * levels], eps); // First local quad, first global quad
CHECK_REAL_EQUAL(234.8390, val[1 * levels], eps); // Median local quad, 2256th global quad
CHECK_REAL_EQUAL(232.6458, val[2 * levels], eps); // Median local quad, 2305th global quad
CHECK_REAL_EQUAL(205.3905, val[3 * levels], eps); // Last local quad, 4560th global quad
}
else if (1 == rank) {
CHECK_REAL_EQUAL(252.7116, val[0 * levels], eps); // First local quad, 49th global quad
CHECK_REAL_EQUAL(232.6670, val[1 * levels], eps); // Median local quad, 2304th global quad
CHECK_REAL_EQUAL(234.6922, val[2 * levels], eps); // Median local quad, 2353th global quad
CHECK_REAL_EQUAL(200.6828, val[3 * levels], eps); // Last local quad, last global quad
}
}
}
void test_read_all()
{
Core moab;
Interface& mb = moab;
std::string opts;
get_options(opts);
// Read mesh and read all variables at all timesteps
ErrorCode rval = mb.load_file(example, 0, opts.c_str());
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
#else
int procs = 1;
#endif
// Make check runs this test on one processor
if (1 == procs) {
// For u, wind and vorticity, check tag values on two entities
double val[2 * layers];
// Check tags for vertex variable u
Tag u_tag0, u_tag1;
rval = mb.tag_get_handle("u0", layers, MB_TYPE_DOUBLE, u_tag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("u1", layers, MB_TYPE_DOUBLE, u_tag1);
CHECK_ERR(rval);
// Get vertices (1280 vertices)
Range verts;
rval = mb.get_entities_by_type(0, MBVERTEX, verts);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)1280, verts.size());
CHECK_EQUAL((size_t)1, verts.psize());
// Check u tag values on first and last vertices
EntityHandle vert_ents[] = {verts[0], verts[1279]};
// Only check first two layers
// Timestep 0
rval = mb.tag_get_data(u_tag0, vert_ents, 2, val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(-4.839992, val[0 * layers], eps);
CHECK_REAL_EQUAL(-3.699257, val[1 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-4.839925, val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-3.699206, val[1 * layers + 1], eps);
// Timestep 1
rval = mb.tag_get_data(u_tag1, vert_ents, 2, val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(-4.712473, val[0 * layers], eps);
CHECK_REAL_EQUAL(-3.601793, val[1 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-4.712409, val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-3.601743, val[1 * layers + 1], eps);
// Check tags for edge variable wind
Tag wind_tag0, wind_tag1;
rval = mb.tag_get_handle("wind0", layers, MB_TYPE_DOUBLE, wind_tag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("wind1", layers, MB_TYPE_DOUBLE, wind_tag1);
CHECK_ERR(rval);
// Get edges (1920 edges)
Range edges;
rval = mb.get_entities_by_type(0, MBEDGE, edges);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)1920, edges.size());
CHECK_EQUAL((size_t)1, edges.psize());
// Check wind tag values on first and last edges
EntityHandle edge_ents[] = {edges[0], edges[1919]};
// Only check first two layers
// Timestep 0
rval = mb.tag_get_data(wind_tag0, edge_ents, 2, val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(-5.081991, val[0 * layers], eps);
CHECK_REAL_EQUAL(-6.420274, val[1 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-5.081781, val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-6.419831, val[1 * layers + 1], eps);
// Timestep 1
rval = mb.tag_get_data(wind_tag1, edge_ents, 2, val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(-4.948097, val[0 * layers], eps);
CHECK_REAL_EQUAL(-6.251121, val[1 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(-4.947892, val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-6.250690, val[1 * layers + 1], eps);
// Check tags for cell variable vorticity
Tag vorticity_tag0, vorticity_tag1;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("vorticity1", layers, MB_TYPE_DOUBLE, vorticity_tag1);
CHECK_ERR(rval);
// Get cells (12 pentagons and 630 hexagons)
Range cells;
rval = mb.get_entities_by_type(0, MBPOLYGON, cells);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)642, cells.size());
// GCRM pentagons are always padded to hexagons
CHECK_EQUAL((size_t)1, cells.psize());
// Check vorticity tag values on first and last cells
EntityHandle cell_ents[] = {cells[0], cells[641]};
// Only check first two layers
// Timestep 0
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 2, val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(3.629994, val[0 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, val[1 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, val[1 * layers + 1], eps);
// Timestep 1
rval = mb.tag_get_data(vorticity_tag1, cell_ents, 2, val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(3.534355, val[0 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, val[1 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.534306, val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, val[1 * layers + 1], eps);
// Check tags for cell variable pressure
Tag pressure_tag0, pressure_tag1;
rval = mb.tag_get_handle("pressure0", interfaces, MB_TYPE_DOUBLE, pressure_tag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("pressure1", interfaces, MB_TYPE_DOUBLE, pressure_tag1);
CHECK_ERR(rval);
// For pressure, check tag values on two cells
double pressure_val[2 * interfaces];
// Check pressure tag values on first and last cells
// Only check first two interfaces
// Timestep 0
rval = mb.tag_get_data(pressure_tag0, cell_ents, 2, pressure_val);
CHECK_ERR(rval);
// Interface 0
CHECK_REAL_EQUAL(4.44234e-06, pressure_val[0 * interfaces], 1e-11);
CHECK_REAL_EQUAL(0.2486804, pressure_val[1 * interfaces], 1e-7);
// Interface 1
CHECK_REAL_EQUAL(4.44234e-06, pressure_val[0 * interfaces + 1], 1e-11);
CHECK_REAL_EQUAL(0.2486804, pressure_val[1 * interfaces + 1], 1e-7);
// Timestep 1
rval = mb.tag_get_data(pressure_tag1, cell_ents, 2, pressure_val);
CHECK_ERR(rval);
// Interface 0
CHECK_REAL_EQUAL(2.365176e-07, pressure_val[0 * interfaces], 1e-13);
CHECK_REAL_EQUAL(0.02234409, pressure_val[1 * interfaces], 1e-8);
// Interface 1
CHECK_REAL_EQUAL(2.365176e-07, pressure_val[0 * interfaces + 1], 1e-13);
CHECK_REAL_EQUAL(0.02234409, pressure_val[1 * interfaces + 1], 1e-8);
}
}
void test_read_onevar()
{
Core moab;
Interface& mb = moab;
std::string opts;
get_options(opts);
// Read mesh and read cell variable vorticity at all timesteps
opts += ";VARIABLE=vorticity";
ErrorCode rval = mb.load_file(example, NULL, opts.c_str());
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
#else
int procs = 1;
#endif
// Make check runs this test on one processor
if (1 == procs) {
// Check vorticity tags
Tag vorticity_tag0, vorticity_tag1;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("vorticity1", layers, MB_TYPE_DOUBLE, vorticity_tag1);
CHECK_ERR(rval);
// Get cells (12 pentagons and 630 hexagons)
Range cells;
rval = mb.get_entities_by_type(0, MBPOLYGON, cells);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)642, cells.size());
// GCRM pentagons are always padded to hexagons
CHECK_EQUAL((size_t)1, cells.psize());
// Check vorticity tag values on 4 cells: first cell, two median cells, and last cell
EntityHandle cell_ents[] = {cells[0], cells[320], cells[321], cells[641]};
double vorticity_val[4 * layers];
// Only check first two layers
// Timestep 0
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 4, vorticity_val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity_val[0 * layers], eps);
CHECK_REAL_EQUAL(0.131688, vorticity_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity_val[2 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity_val[3 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(0.131686, vorticity_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity_val[2 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity_val[3 * layers + 1], eps);
// Timestep 1
rval = mb.tag_get_data(vorticity_tag1, cell_ents, 4, vorticity_val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(3.534355, vorticity_val[0 * layers], eps);
CHECK_REAL_EQUAL(0.128218, vorticity_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, vorticity_val[2 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, vorticity_val[3 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.534306, vorticity_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(0.128216, vorticity_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, vorticity_val[2 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, vorticity_val[3 * layers + 1], eps);
}
}
void test_read_novars()
{
Core moab;
Interface& mb = moab;
// Need a file set for nomesh to work right
EntityHandle file_set;
ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set);
CHECK_ERR(rval);
std::string orig, opts;
get_options(orig);
CHECK_ERR(rval);
// Read header info only, no mesh, no variables
opts = orig + ";NOMESH;VARIABLE=";
rval = mb.load_file(example, &file_set, opts.c_str());
CHECK_ERR(rval);
// Read mesh, but still no variables
opts = orig + ";VARIABLE=";
rval = mb.load_file(example, &file_set, opts.c_str());
CHECK_ERR(rval);
// Check vorticity tags
Tag vorticity_tag0, vorticity_tag1;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0);
// Tag vorticity0 should not exist
CHECK_EQUAL(rval, MB_TAG_NOT_FOUND);
rval = mb.tag_get_handle("vorticity1", layers, MB_TYPE_DOUBLE, vorticity_tag1);
// Tag vorticity1 should not exist
CHECK_EQUAL(rval, MB_TAG_NOT_FOUND);
// Read vorticity at 1st timestep, no need to read mesh
opts = orig + ";VARIABLE=vorticity;TIMESTEP=0;NOMESH";
rval = mb.load_file(example, &file_set, opts.c_str());
CHECK_ERR(rval);
// Check vorticity tags again
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0);
// Tag vorticity0 should exist at this time
CHECK_ERR(rval);
// Tag vorticity1 should still not exist
rval = mb.tag_get_handle("vorticity1", layers, MB_TYPE_DOUBLE, vorticity_tag1);
CHECK_EQUAL(rval, MB_TAG_NOT_FOUND);
// Read vorticity at 2nd timestep, no need to read mesh
opts = orig + ";VARIABLE=vorticity;TIMESTEP=1;NOMESH";
rval = mb.load_file(example, &file_set, opts.c_str());
CHECK_ERR(rval);
// Check tag vorticity1 again
rval = mb.tag_get_handle("vorticity1", layers, MB_TYPE_DOUBLE, vorticity_tag1);
// Tag vorticity1 should exist at this time
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int procs = pcomm->proc_config().proc_size();
#else
int procs = 1;
#endif
// Make check runs this test on one processor
if (1 == procs) {
// Get cells (12 pentagons and 630 hexagons)
Range cells;
rval = mb.get_entities_by_type(file_set, MBPOLYGON, cells);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)642, cells.size());
// GCRM pentagons are always padded to hexagons
CHECK_EQUAL((size_t)1, cells.psize());
// Check vorticity tag values on 4 cells: first cell, two median cells, and last cell
EntityHandle cell_ents[] = {cells[0], cells[320], cells[321], cells[641]};
double vorticity_val[4 * layers];
// Only check first two layers
// Timestep 0
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 4, vorticity_val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity_val[0 * layers], eps);
CHECK_REAL_EQUAL(0.131688, vorticity_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity_val[2 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity_val[3 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(0.131686, vorticity_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity_val[2 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity_val[3 * layers + 1], eps);
// Timestep 1
rval = mb.tag_get_data(vorticity_tag1, cell_ents, 4, vorticity_val);
CHECK_ERR(rval);
// Layer 0
CHECK_REAL_EQUAL(3.534355, vorticity_val[0 * layers], eps);
CHECK_REAL_EQUAL(0.128218, vorticity_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, vorticity_val[2 * layers], eps);
CHECK_REAL_EQUAL(-0.540269, vorticity_val[3 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.534306, vorticity_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(0.128216, vorticity_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, vorticity_val[2 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.540262, vorticity_val[3 * layers + 1], eps);
}
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
std::string extra_read_opts;
std::string fileN= TestDir + "/io/mpasx1.642.t.2.nc";
const char *filename_mesh1 = fileN.c_str();
bool flux_form = false;
if (argc > 1)
{
int index = 1;
while (index < argc)
{
if (!strcmp(argv[index], "-gtol")) // this is for geometry tolerance
{
gtol = atof(argv[++index]);
}
if (!strcmp(argv[index], "-dt"))
{
delta_t = atof(argv[++index]);
}
if (!strcmp(argv[index], "-input"))
{
filename_mesh1 = argv[++index];
}
if (!strcmp(argv[index], "-R"))
{
radius = atof(argv[++index]);
}
if (!strcmp(argv[index], "-O"))
{
extra_read_opts = std::string(argv[++index]);
}
if (!strcmp(argv[index], "-FF"))
{
flux_form= true;
}
if (!strcmp(argv[index], "-v"))
{
Verbose = true;
}
if (!strcmp(argv[index], "-t"))
{
t = atof(argv[++index]);
}
if (!strcmp(argv[index], "-t"))
{
t = atof(argv[++index]);
}
if (!strcmp(argv[index], "-rot"))
{
rot = atof(argv[++index]);
rot = M_PI/rot; // so rot 50 means rotate with M_PI/50 radians
}
index++;
}
}
// start copy
std::string opts = std::string("PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN")+
std::string(";PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=;NO_EDGES;")+extra_read_opts;
Core moab;
Interface & mb = moab;
EntityHandle euler_set;
ErrorCode rval;
rval = mb.create_meshset(MESHSET_SET, euler_set);
CHECK_ERR(rval);
clock_t tt = clock();
rval = mb.load_file(filename_mesh1, &euler_set, opts.c_str());
CHECK_ERR(rval);
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
CHECK_ERR(rval);
/*rval = pcomm->check_all_shared_handles();
CHECK_ERR(rval);*/
// end copy
int rank = pcomm->proc_config().proc_rank();
int procs = pcomm->proc_config().proc_size();
if (0==rank)
std::cout << " case 1: use -gtol " << gtol << " -dt " << delta_t <<
" -R " << radius << " -input " << filename_mesh1 << " -t " << t << " -rot " << rot << "\n";
if (0==rank)
{
std::cout << "load mesh from " << filename_mesh1 << "\n on " << procs << " processors in "
<< (clock() - tt) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
tt = clock();
}
rval = manufacture_lagrange_mesh_on_sphere(&mb, euler_set);
if (MB_SUCCESS != rval)
return 1;
// create a set with quads corresponding to each initial edge spanned with the displacement field
if (flux_form)
{
rval = create_span_quads(&mb, euler_set, rank);
if (MB_SUCCESS != rval)
return 1;
}
EntityHandle covering_lagr_set;
rval = mb.create_meshset(MESHSET_SET, covering_lagr_set);
CHECK_ERR(rval);
Intx2MeshOnSphere worker(&mb);
//double radius = 1.; // input
worker.SetRadius(radius);
if (0==rank)
{
std::cout << "manufacture departure mesh " << filename_mesh1 << "\n on " << procs << " processors in "
<< (clock() - tt) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
tt = clock();
}
worker.SetErrorTolerance(gtol);
rval = worker.create_departure_mesh_2nd_alg(euler_set, covering_lagr_set);
CHECK_ERR(rval);
if (0==rank)
{
std::cout << "communicate covering mesh on " << procs << " processors in "
<< (clock() - tt) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
tt = clock();
}
if (Verbose)
{
std::stringstream lagrIni;
lagrIni<<"lagr0" << rank<<".h5m";
rval = mb.write_file(lagrIni.str().c_str(), 0, 0, &covering_lagr_set, 1);
}
rval = enforce_convexity(&mb, covering_lagr_set, rank);
if (MB_SUCCESS != rval)
return 1;
if (Verbose)
{
std::stringstream ste;
ste<<"euler0" << rank<<".h5m";
rval = mb.write_file(ste.str().c_str(), 0, 0, &euler_set, 1);
}
if (MB_SUCCESS != rval)
std::cout << "can't write lagr set\n";
EntityHandle outputSet;
rval = mb.create_meshset(MESHSET_SET, outputSet);
if (MB_SUCCESS != rval)
return 1;
rval = worker.intersect_meshes(covering_lagr_set, euler_set, outputSet);
if (MB_SUCCESS != rval)
return 1;
if (0==rank)
{
std::cout << "intersect meshes in " << procs << " processors in "
<< (clock() - tt) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
tt = clock();
}
if (Verbose && rank<=4)
{
std::string opts_write("");
std::stringstream outf;
outf << "intersect0" << rank << ".h5m";
rval = mb.write_file(outf.str().c_str(), 0, 0, &outputSet, 1);
if (MB_SUCCESS != rval)
std::cout << "can't write output\n";
}
if (rank <= 4)
{
double intx_area = area_on_sphere_lHuiller(&mb, outputSet, radius);
double arrival_area = area_on_sphere_lHuiller(&mb, euler_set, radius);
std::cout << "On proc " << rank << " arrival area: " << arrival_area
<< " intersection area:" << intx_area << " rel error: "
<< fabs((intx_area - arrival_area) / arrival_area) << "\n";
}
MPI_Finalize();
if (MB_SUCCESS != rval)
return 1;
return 0;
}
ErrorCode DeformMeshRemap::execute()
{
// Read master/slave files and get fluid/solid material sets
ErrorCode rval = read_file(MASTER, masterFileName, masterSet);MB_CHK_ERR(rval);
if (solidSetNos[MASTER].empty() || fluidSetNos[MASTER].empty()) {
rval = find_other_sets(MASTER, masterSet);MB_CHK_SET_ERR(rval, "Failed to find other sets in master mesh");
}
bool have_slave = !(slaveFileName == "none");
if (have_slave) {
rval = read_file(SLAVE, slaveFileName, slaveSet);MB_CHK_ERR(rval);
if (solidSetNos[SLAVE].empty() || fluidSetNos[SLAVE].empty()) {
rval = find_other_sets(SLAVE, slaveSet);MB_CHK_SET_ERR(rval, "Failed to find other sets in slave mesh");
}
}
if (debug) cout << "Constructing data coupler/search tree on master mesh..." << endl;
Range src_elems = solidElems[MASTER];
src_elems.merge(fluidElems[MASTER]);
// Initialize data coupler on source elements
DataCoupler dc_master(mbImpl, src_elems, 0, NULL);
Range tgt_verts;
if (have_slave) {
// Locate slave vertices in master, orig coords; do this with a data coupler, so you can
// later interpolate
Range tmp_range = solidElems[SLAVE];
tmp_range.merge(fluidElems[SLAVE]);
rval = mbImpl->get_adjacencies(tmp_range, 0, false, tgt_verts, Interface::UNION);MB_CHK_SET_ERR(rval, "Failed to get target verts");
// Locate slave vertices, caching results in dc
if (debug) cout << "Locating slave vertices in master mesh..." << endl;
rval = dc_master.locate_points(tgt_verts);MB_CHK_SET_ERR(rval, "Point location of tgt verts failed");
int num_located = dc_master.spatial_locator()->local_num_located();
if (num_located != (int)tgt_verts.size()) {
rval = MB_FAILURE;
cout << "Only " << num_located << " out of " << tgt_verts.size() << " target points successfully located." << endl;
return rval;
}
}
// Deform the master's solid mesh, put results in a new tag
if (debug) cout << "Deforming fluid elements in master mesh..." << endl;
rval = deform_master(fluidElems[MASTER], solidElems[MASTER], "xnew");MB_CHK_ERR(rval);
{ // To isolate the lloyd smoother & delete when done
if (debug) {
// Output the skin of smoothed elems, as a check
// Get the skin; get facets, because we might need to filter on shared entities
Skinner skinner(mbImpl);
Range skin;
rval = skinner.find_skin(0, fluidElems[MASTER], false, skin);MB_CHK_SET_ERR(rval, "Unable to find skin");
EntityHandle skin_set;
cout << "Writing skin_mesh.g and fluid_mesh.g." << endl;
rval = mbImpl->create_meshset(MESHSET_SET, skin_set);MB_CHK_SET_ERR(rval, "Failed to create skin set");
rval = mbImpl->add_entities(skin_set, skin);MB_CHK_SET_ERR(rval, "Failed to add skin entities to set");
rval = mbImpl->write_file("skin_mesh.vtk", NULL, NULL, &skin_set, 1);MB_CHK_SET_ERR(rval, "Failure to write skin set");
rval = mbImpl->remove_entities(skin_set, skin);MB_CHK_SET_ERR(rval, "Failed to remove skin entities from set");
rval = mbImpl->add_entities(skin_set, fluidElems[MASTER]);MB_CHK_SET_ERR(rval, "Failed to add fluid entities to set");
rval = mbImpl->write_file("fluid_mesh.vtk", NULL, NULL, &skin_set, 1);MB_CHK_SET_ERR(rval, "Failure to write fluid set");
rval = mbImpl->delete_entities(&skin_set, 1);MB_CHK_SET_ERR(rval, "Failed to delete skin set");
}
// Smooth the master mesh
if (debug) cout << "Smoothing fluid elements in master mesh..." << endl;
LloydSmoother ll(mbImpl, NULL, fluidElems[MASTER], xNew);
rval = ll.perform_smooth();MB_CHK_SET_ERR(rval, "Failed in lloyd smoothing");
cout << "Lloyd smoothing required " << ll.num_its() << " iterations." << endl;
}
// Transfer xNew to coords, for master
if (debug) cout << "Transferring coords tag to vertex coordinates in master mesh..." << endl;
rval = write_to_coords(solidElems[MASTER], xNew);MB_CHK_SET_ERR(rval, "Failed writing tag to master fluid verts");
rval = write_to_coords(fluidElems[MASTER], xNew);MB_CHK_SET_ERR(rval, "Failed writing tag to master fluid verts");
if (have_slave) {
// Map new locations to slave
// Interpolate xNew to slave points
if (debug) cout << "Interpolating new coordinates to slave vertices..." << endl;
rval = dc_master.interpolate((int)DataCoupler::VOLUME, "xnew");MB_CHK_SET_ERR(rval, "Failed to interpolate target solution");
// Transfer xNew to coords, for slave
if (debug) cout << "Transferring coords tag to vertex coordinates in slave mesh..." << endl;
rval = write_to_coords(tgt_verts, xNew);MB_CHK_SET_ERR(rval, "Failed writing tag to slave verts");
}
if (debug) {
string str;
#ifdef USE_MPI
if (pcMaster && pcMaster->size() > 1)
str = "PARALLEL=WRITE_PART";
#endif
if (debug) cout << "Writing smoothed_master.h5m..." << endl;
rval = mbImpl->write_file("smoothed_master.h5m", NULL, str.c_str(), &masterSet, 1);
if (have_slave) {
#ifdef USE_MPI
str.clear();
if (pcSlave && pcSlave->size() > 1)
str = "PARALLEL=WRITE_PART";
#endif
if (debug) cout << "Writing slave_interp.h5m..." << endl;
rval = mbImpl->write_file("slave_interp.h5m", NULL, str.c_str(), &slaveSet, 1);
} // if have_slave
} // if debug
if (debug)
dc_master.spatial_locator()->get_tree()->tree_stats().print();
return MB_SUCCESS;
}
void test_gather_onevar()
{
Core moab;
Interface& mb = moab;
EntityHandle file_set;
ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set);
CHECK_ERR(rval);
std::string opts;
get_options(opts);
// Read vertex variable T and create gather set on processor 0
opts += ";VARIABLE=T;GATHER_SET=0";
#ifdef MOAB_HAVE_MPI
opts += ";PARALLEL_RESOLVE_SHARED_ENTS";
#endif
rval = mb.load_file(example, &file_set, opts.c_str());
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int rank = pcomm->proc_config().proc_rank();
Range verts, verts_owned;
rval = mb.get_entities_by_type(file_set, MBVERTEX, verts);
CHECK_ERR(rval);
// Get local owned vertices
rval = pcomm->filter_pstatus(verts, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &verts_owned);
CHECK_ERR(rval);
EntityHandle gather_set = 0;
if (0 == rank) {
// Get gather set
ReadUtilIface* readUtilIface;
mb.query_interface(readUtilIface);
rval = readUtilIface->get_gather_set(gather_set);
CHECK_ERR(rval);
assert(gather_set != 0);
}
Tag Ttag0, gid_tag;
rval = mb.tag_get_handle("T0", levels, MB_TYPE_DOUBLE, Ttag0, MB_TAG_DENSE);
CHECK_ERR(rval);
rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE);
CHECK_ERR(rval);
pcomm->gather_data(verts_owned, Ttag0, gid_tag, gather_set, 0);
if (0 == rank) {
// Get gather set vertices
Range gather_set_verts;
rval = mb.get_entities_by_type(gather_set, MBVERTEX, gather_set_verts);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)3458, gather_set_verts.size());
// Get T0 tag values on 4 strategically selected gather set vertices
double T0_val[4 * levels];
EntityHandle vert_ents[] = {gather_set_verts[0], gather_set_verts[1728],
gather_set_verts[1729], gather_set_verts[3457]};
rval = mb.tag_get_data(Ttag0, vert_ents, 4, T0_val);
CHECK_ERR(rval);
const double eps = 0.001;
// Check first level values
CHECK_REAL_EQUAL(233.1136, T0_val[0 * levels], eps); // First vert
CHECK_REAL_EQUAL(236.1505, T0_val[1 * levels], eps); // Median vert
CHECK_REAL_EQUAL(235.7722, T0_val[2 * levels], eps); // Median vert
CHECK_REAL_EQUAL(234.0416, T0_val[3 * levels], eps); // Last vert
}
#endif
}
ErrorCode get_max_volume(Core &mb, EntityHandle fileset, int dim, double &vmax)
{
ErrorCode error;
VerdictWrapper vw(&mb);
QualityType q;
switch (dim) {
case 1: q = MB_LENGTH; break;
case 2: q = MB_AREA; break;
case 3: q = MB_VOLUME; break;
default: return MB_FAILURE; break;
}
//Get all entities of the highest dimension which is passed as a command line argument.
Range allents, owned;
error = mb.get_entities_by_handle(fileset, allents);MB_CHK_ERR(error);
owned = allents.subset_by_dimension(dim);MB_CHK_ERR(error);
//Get all owned entities
#ifdef MOAB_HAVE_MPI
int size = 1;
MPI_Comm_size( MPI_COMM_WORLD, &size );
int mpi_err;
if (size>1)
{
// filter the entities not owned, so we do not process them more than once
ParallelComm* pcomm = moab::ParallelComm::get_pcomm(&mb, 0);
Range current = owned;
owned.clear();
error = pcomm->filter_pstatus(current, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &owned);
if (error != MB_SUCCESS)
{
MPI_Finalize();
return MB_FAILURE;
}
}
#endif
double vmax_local=0;
//Find the maximum volume of an entity in the owned mesh
for (Range::iterator it=owned.begin(); it != owned.end(); it++)
{
double volume;
error = vw.quality_measure(*it, q, volume);MB_CHK_ERR(error);
if (volume >vmax_local)
vmax_local = volume;
}
//Get the global maximum
double vmax_global = vmax_local;
#ifdef MOAB_HAVE_MPI
mpi_err = MPI_Reduce(&vmax_local, &vmax_global, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (mpi_err)
{
MPI_Finalize();
return MB_FAILURE;
}
#endif
vmax = vmax_global;
return MB_SUCCESS;
}
void test_read_fv_onevar()
{
Core moab;
Interface& mb = moab;
std::string opts;
ErrorCode rval = get_options(opts);
CHECK_ERR(rval);
opts += std::string(";VARIABLE=T");
rval = mb.load_file(example_fv, NULL, opts.c_str());
CHECK_ERR(rval);
// Check for proper tags
Tag Ttag0, Ttag1;
rval = mb.tag_get_handle("T0", levels, MB_TYPE_DOUBLE, Ttag0);
CHECK_ERR(rval);
rval = mb.tag_get_handle("T1", levels, MB_TYPE_DOUBLE, Ttag1);
CHECK_ERR(rval);
// Check values of tag T0 (first level) at some strategically chosen places below
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int rank = pcomm->proc_config().proc_rank();
int procs = pcomm->proc_config().proc_size();
#else
int rank = 0;
int procs = 1;
#endif
const double eps = 0.0001;
double val[8 * levels];
if (1 == procs) {
Range global_quads;
rval = mb.get_entities_by_type(0, MBQUAD, global_quads);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)3312, global_quads.size());
EntityHandle gloabl_quad_ents[] = {global_quads[0], global_quads[1619], global_quads[1656], global_quads[3275],
global_quads[36], global_quads[1655], global_quads[1692], global_quads[3311]};
rval = mb.tag_get_data(Ttag0, &gloabl_quad_ents[0], 8, val);
CHECK_REAL_EQUAL(253.6048, val[0 * levels], eps); // First global quad
CHECK_REAL_EQUAL(232.2170, val[1 * levels], eps); // 1620th global quad
CHECK_REAL_EQUAL(232.7454, val[2 * levels], eps); // 1657th global quad
CHECK_REAL_EQUAL(210.2581, val[3 * levels], eps); // 3276th global quad
CHECK_REAL_EQUAL(253.6048, val[4 * levels], eps); // 37th global quad
CHECK_REAL_EQUAL(232.9553, val[5 * levels], eps); // 1656th global quad
CHECK_REAL_EQUAL(232.1704, val[6 * levels], eps); // 1693th global quad
CHECK_REAL_EQUAL(210.2581, val[7 * levels], eps); // Last global quad
}
else if (2 == procs) {
Range local_quads;
rval = mb.get_entities_by_type(0, MBQUAD, local_quads);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)1656, local_quads.size());
EntityHandle local_quad_ents[] = {local_quads[0], local_quads[827], local_quads[828], local_quads[1655]};
rval = mb.tag_get_data(Ttag0, &local_quad_ents[0], 4, val);
if (0 == rank) {
CHECK_REAL_EQUAL(253.6048, val[0 * levels], eps); // First local quad, first global quad
CHECK_REAL_EQUAL(232.2170, val[1 * levels], eps); // Median local quad, 1620th global quad
CHECK_REAL_EQUAL(232.7454, val[2 * levels], eps); // Median local quad, 1657th global quad
CHECK_REAL_EQUAL(210.2581, val[3 * levels], eps); // Last local quad, 3276th global quad
}
else if (1 == rank) {
CHECK_REAL_EQUAL(253.6048, val[0 * levels], eps); // First local quad, 37th global quad
CHECK_REAL_EQUAL(232.9553, val[1 * levels], eps); // Median local quad, 1656th global quad
CHECK_REAL_EQUAL(232.1704, val[2 * levels], eps); // Median local quad, 1693th global quad
CHECK_REAL_EQUAL(210.2581, val[3 * levels], eps); // Last local quad, last global quad
}
}
}
ErrorCode DeformMeshRemap::read_file(int m_or_s, string &fname, EntityHandle &seth)
{
// Create meshset
ErrorCode rval = mbImpl->create_meshset(0, seth);MB_CHK_SET_ERR(rval, "Couldn't create master/slave set");
ostringstream options;
#ifdef USE_MPI
ParallelComm *pc = (m_or_s == MASTER ? pcMaster : pcSlave);
if (pc && pc->size() > 1) {
if (debug) options << "DEBUG_IO=1;CPUTIME;";
options << "PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS;"
<< "PARALLEL_GHOSTS=2.0.1;PARALLEL_COMM=" << pc->get_id();
}
#endif
rval = mbImpl->load_file(fname.c_str(), &seth, options.str().c_str());MB_CHK_SET_ERR(rval, "Couldn't load master/slave mesh");
if (*solidSetNos[m_or_s].begin() == -1 || *fluidSetNos[m_or_s].begin() == -1) return MB_SUCCESS;
// Get material sets for solid/fluid
Tag tagh;
rval = mbImpl->tag_get_handle(MATERIAL_SET_TAG_NAME, tagh);MB_CHK_SET_ERR(rval, "Couldn't get material set tag name");
for (set<int>::iterator sit = solidSetNos[m_or_s].begin(); sit != solidSetNos[m_or_s].end(); ++sit) {
Range sets;
int set_no = *sit;
const void *setno_ptr = &set_no;
rval = mbImpl->get_entities_by_type_and_tag(seth, MBENTITYSET, &tagh, &setno_ptr, 1, sets);
if (MB_SUCCESS != rval || sets.empty()) {
MB_SET_ERR(MB_FAILURE, "Couldn't find solid set #" << *sit);
}
else
solidSets[m_or_s].merge(sets);
}
// Get solid entities, and dimension
Range tmp_range;
for (Range::iterator rit = solidSets[m_or_s].begin(); rit != solidSets[m_or_s].end(); ++rit) {
rval = mbImpl->get_entities_by_handle(*rit, tmp_range, true);MB_CHK_SET_ERR(rval, "Failed to get entities in solid");
}
if (!tmp_range.empty()) {
int dim = mbImpl->dimension_from_handle(*tmp_range.rbegin());
assert(dim > 0 && dim < 4);
solidElems[m_or_s] = tmp_range.subset_by_dimension(dim);
}
if (debug)
cout << "Read " << solidElems[m_or_s].size() << " solid elements from " << solidSets[m_or_s].size() <<
" sets in " << (m_or_s == MASTER ? "master" : "slave") << " mesh." << endl;
for (set<int>::iterator sit = fluidSetNos[m_or_s].begin(); sit != fluidSetNos[m_or_s].end(); ++sit) {
Range sets;
int set_no = *sit;
const void *setno_ptr = &set_no;
rval = mbImpl->get_entities_by_type_and_tag(seth, MBENTITYSET, &tagh, &setno_ptr, 1, sets);
if (MB_SUCCESS != rval || sets.empty()) {
MB_SET_ERR(MB_FAILURE, "Couldn't find fluid set #" << *sit);
}
else
fluidSets[m_or_s].merge(sets);
}
// Get fluid entities, and dimension
tmp_range.clear();
for (Range::iterator rit = fluidSets[m_or_s].begin(); rit != fluidSets[m_or_s].end(); ++rit) {
rval = mbImpl->get_entities_by_handle(*rit, tmp_range, true);MB_CHK_SET_ERR(rval, "Failed to get entities in fluid");
}
if (!tmp_range.empty()) {
int dim = mbImpl->dimension_from_handle(*tmp_range.rbegin());
assert(dim > 0 && dim < 4);
fluidElems[m_or_s] = tmp_range.subset_by_dimension(dim);
}
if (debug)
cout << "Read " << fluidElems[m_or_s].size() << " fluid elements from " << fluidSets[m_or_s].size() <<
" sets in " << (m_or_s == MASTER ? "master" : "slave") << " mesh." << endl;
return rval;
}
void test_gather_onevar()
{
Core moab;
Interface& mb = moab;
EntityHandle file_set;
ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set);
CHECK_ERR(rval);
std::string opts;
get_options(opts);
// Read cell variable vorticity and create gather set on processor 0
opts += ";VARIABLE=vorticity;GATHER_SET=0";
rval = mb.load_file(example, &file_set, opts.c_str());
CHECK_ERR(rval);
#ifdef MOAB_HAVE_MPI
ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
int rank = pcomm->proc_config().proc_rank();
Range cells, cells_owned;
rval = mb.get_entities_by_type(file_set, MBPOLYGON, cells);
CHECK_ERR(rval);
// Get local owned cells
rval = pcomm->filter_pstatus(cells, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &cells_owned);
CHECK_ERR(rval);
EntityHandle gather_set = 0;
if (0 == rank) {
// Get gather set
ReadUtilIface* readUtilIface;
mb.query_interface(readUtilIface);
rval = readUtilIface->get_gather_set(gather_set);
CHECK_ERR(rval);
assert(gather_set != 0);
}
Tag vorticity_tag0, gid_tag;
rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0, MB_TAG_DENSE);
CHECK_ERR(rval);
rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE);
CHECK_ERR(rval);
pcomm->gather_data(cells_owned, vorticity_tag0, gid_tag, gather_set, 0);
if (0 == rank) {
// Get gather set cells
Range gather_set_cells;
rval = mb.get_entities_by_type(gather_set, MBPOLYGON, gather_set_cells);
CHECK_ERR(rval);
CHECK_EQUAL((size_t)642, gather_set_cells.size());
CHECK_EQUAL((size_t)1, gather_set_cells.psize());
// Check vorticity0 tag values on 4 gather set cells: first cell, two median cells, and last cell
EntityHandle cell_ents[] = {gather_set_cells[0], gather_set_cells[320],
gather_set_cells[321], gather_set_cells[641]};
double vorticity0_val[4 * layers];
rval = mb.tag_get_data(vorticity_tag0, cell_ents, 4, vorticity0_val);
CHECK_ERR(rval);
// Only check first two layers
// Layer 0
CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps);
CHECK_REAL_EQUAL(0.131688, vorticity0_val[1 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps);
CHECK_REAL_EQUAL(-0.554888, vorticity0_val[3 * layers], eps);
// Layer 1
CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps);
CHECK_REAL_EQUAL(0.131686, vorticity0_val[1 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps);
CHECK_REAL_EQUAL(-0.554881, vorticity0_val[3 * layers + 1], eps);
}
#endif
}
int main(int argc, char **argv)
{
#ifdef MOAB_HAVE_MPI
MPI_Init(&argc, &argv);
string options;
// Need option handling here for input filename
if (argc > 1) {
// User has input a mesh file
test_file_name = argv[1];
}
int nbComms = 1;
if (argc > 2)
nbComms = atoi(argv[2]);
options = "PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS";
// Get MOAB instance
Interface* mb = new (std::nothrow) Core;
if (NULL == mb)
return 1;
MPI_Comm comm;
int global_rank, global_size;
MPI_Comm_rank(MPI_COMM_WORLD, &global_rank);
MPI_Comm_rank(MPI_COMM_WORLD, &global_size);
int color = global_rank % nbComms; // For each angle group a different color
if (nbComms > 1) {
// Split the communicator, into ngroups = nbComms
MPI_Comm_split(MPI_COMM_WORLD, color, global_rank, &comm);
}
else
comm = MPI_COMM_WORLD;
// Get the ParallelComm instance
ParallelComm* pcomm = new ParallelComm(mb, comm);
int nprocs = pcomm->proc_config().proc_size();
int rank = pcomm->proc_config().proc_rank();
#ifndef NDEBUG
MPI_Comm rcomm = pcomm->proc_config().proc_comm();
assert(rcomm == comm);
#endif
if (0 == global_rank)
cout << " global rank:" << global_rank << " color:" << color << " rank:" << rank << " of " << nprocs << " processors\n";
if (1 == global_rank)
cout << " global rank:" << global_rank << " color:" << color << " rank:" << rank << " of " << nprocs << " processors\n";
MPI_Barrier(MPI_COMM_WORLD);
if (0 == global_rank)
cout << "Reading file " << test_file_name << "\n with options: " << options <<
"\n on " << nprocs << " processors on " << nbComms << " communicator(s)\n";
// Read the file with the specified options
ErrorCode rval = mb->load_file(test_file_name.c_str(), 0, options.c_str());MB_CHK_ERR(rval);
Range shared_ents;
// Get entities shared with all other processors
rval = pcomm->get_shared_entities(-1, shared_ents);MB_CHK_ERR(rval);
// Filter shared entities with not not_owned, which means owned
Range owned_entities;
rval = pcomm->filter_pstatus(shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &owned_entities);MB_CHK_ERR(rval);
unsigned int nums[4] = {0}; // to store the owned entities per dimension
for (int i = 0; i < 4; i++)
nums[i] = (int)owned_entities.num_of_dimension(i);
vector<int> rbuf(nprocs*4, 0);
MPI_Gather(nums, 4, MPI_INT, &rbuf[0], 4, MPI_INT, 0, comm);
// Print the stats gathered:
if (0 == global_rank) {
for (int i = 0; i < nprocs; i++)
cout << " Shared, owned entities on proc " << i << ": " << rbuf[4*i] << " verts, " <<
rbuf[4*i + 1] << " edges, " << rbuf[4*i + 2] << " faces, " << rbuf[4*i + 3] << " elements" << endl;
}
// Now exchange 1 layer of ghost elements, using vertices as bridge
// (we could have done this as part of reading process, using the PARALLEL_GHOSTS read option)
rval = pcomm->exchange_ghost_cells(3, // int ghost_dim
0, // int bridge_dim
1, // int num_layers
0, // int addl_ents
true);MB_CHK_ERR(rval); // bool store_remote_handles
// Repeat the reports, after ghost exchange
shared_ents.clear();
owned_entities.clear();
rval = pcomm->get_shared_entities(-1, shared_ents);MB_CHK_ERR(rval);
rval = pcomm->filter_pstatus(shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &owned_entities);MB_CHK_ERR(rval);
// Find out how many shared entities of each dimension are owned on this processor
for (int i = 0; i < 4; i++)
nums[i] = (int)owned_entities.num_of_dimension(i);
// Gather the statistics on processor 0
MPI_Gather(nums, 4, MPI_INT, &rbuf[0], 4, MPI_INT, 0, comm);
if (0 == global_rank) {
cout << " \n\n After exchanging one ghost layer: \n";
for (int i = 0; i < nprocs; i++) {
cout << " Shared, owned entities on proc " << i << ": " << rbuf[4*i] << " verts, " <<
rbuf[4*i + 1] << " edges, " << rbuf[4*i + 2] << " faces, " << rbuf[4*i + 3] << " elements" << endl;
}
}
delete mb;
MPI_Finalize();
#else
std::cout<<" compile with MPI and hdf5 for this example to work\n";
#endif
return 0;
}
/// Construct an evaluator.
RefinerTagManager::RefinerTagManager( Interface* in_mesh, Interface* out_mesh )
: shared_procs_in( 5 * MAX_SHARING_PROCS, -1 ), shared_procs_out( MAX_SHARING_PROCS, -1 )
{
assert( in_mesh );
if ( ! out_mesh )
out_mesh = in_mesh;
this->input_mesh = in_mesh;
this->output_mesh = out_mesh;
this->reset_vertex_tags();
this->reset_element_tags();
ParallelComm* ipcomm = ParallelComm::get_pcomm( this->input_mesh, 0 );
ParallelComm* opcomm = 0;
if ( this->output_mesh != this->input_mesh )
{
opcomm = ParallelComm::get_pcomm( this->output_mesh, 0 );
if ( ! opcomm )
{
#ifdef MB_DEBUG
std::cout << "Creating opcomm: " << opcomm << "\n";
#endif // MB_DEBUG
opcomm = new ParallelComm( this->output_mesh, MPI_COMM_WORLD );
}
}
else
{
opcomm = ipcomm;
}
if ( ipcomm )
{
ipcomm->get_shared_proc_tags(
this->tag_ipsproc, this->tag_ipsprocs,
this->tag_ipshand, this->tag_ipshands,
this->tag_ipstatus );
}
else
{
this->tag_ipsproc = this->tag_ipsprocs = 0;
this->tag_ipshand = this->tag_ipshands = 0;
this->tag_ipstatus = 0;
}
if ( opcomm )
{
opcomm->get_shared_proc_tags(
this->tag_opsproc, this->tag_opsprocs,
this->tag_opshand, this->tag_opshands,
this->tag_opstatus );
}
else
{
this->tag_opsproc = this->tag_opsprocs = 0;
this->tag_opshand = this->tag_opshands = 0;
this->tag_opstatus = 0;
}
this->rank =
ipcomm ? ipcomm->proc_config().proc_rank() :
( opcomm ? opcomm->proc_config().proc_rank() : 0 );
// Create the mesh global ID tags if they aren't already there.
int zero = 0;
ErrorCode result;
result = this->input_mesh->tag_get_handle(
GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, this->tag_igid, MB_TAG_DENSE|MB_TAG_CREAT, &zero );
if ( result != MB_SUCCESS )
{
throw new std::logic_error( "Unable to find input mesh global ID tag \"" GLOBAL_ID_TAG_NAME "\"" );
}
result = this->output_mesh->tag_get_handle(
GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, this->tag_ogid, MB_TAG_DENSE|MB_TAG_CREAT, &zero );
if ( result != MB_SUCCESS )
{
throw new std::logic_error( "Unable to find/create output mesh global ID tag \"" GLOBAL_ID_TAG_NAME "\"" );
}
#ifdef MB_DEBUG
std::cout
<< "psproc: " << this->tag_ipsproc << ", " << this->tag_opsproc << "\n"
<< "psprocs: " << this->tag_ipsprocs << ", " << this->tag_opsprocs << "\n"
<< "pshand: " << this->tag_ipshand << ", " << this->tag_opshand << "\n"
<< "pshands: " << this->tag_ipshands << ", " << this->tag_opshands << "\n"
<< "pstatus: " << this->tag_ipstatus << ", " << this->tag_opstatus << "\n"
<< "gid: " << this->tag_igid << ", " << this->tag_ogid << "\n";
#endif // MB_DEBUG
}
