TEST(SimpleMPI,persistent_distributed_array_mpi) { sip::GlobalState::reset_program_count(); sip::SIPMPIAttr &sip_mpi_attr = sip::SIPMPIAttr::get_instance(); int my_rank = sip_mpi_attr.global_rank(); std::cout << "****************************************\n"; sip::DataManager::scope_count=0; //create setup_file std::string job("persistent_distributed_array_mpi"); std::cout << "JOBNAME = " << job << std::endl; double x = 3.456; int norb = 2; int segs[] = {2,3}; if (attr.global_rank() == 0) { init_setup(job.c_str()); set_scalar("x",x); set_constant("norb",norb); std::string tmp = job + "1.siox"; const char* nm= tmp.c_str(); add_sial_program(nm); std::string tmp1 = job + "2.siox"; const char* nm1= tmp1.c_str(); add_sial_program(nm1); set_aoindex_info(2,segs); finalize_setup(); } sip::SIPMPIUtils::check_err(MPI_Barrier(MPI_COMM_WORLD)); setup::BinaryInputFile setup_file(job + ".dat"); setup::SetupReader setup_reader(setup_file); std::cout << "SETUP READER DATA:\n" << setup_reader<< std::endl; //get siox name from setup, load and print the sip tables std::string prog_name = setup_reader.sial_prog_list_.at(0); std::string siox_dir(dir_name); setup::BinaryInputFile siox_file(siox_dir + prog_name); sip::SipTables sipTables(setup_reader, siox_file); if (!sip_mpi_attr.is_server()) { std::cout << "SIP TABLES" << '\n' << sipTables << std::endl; } if (sip_mpi_attr.global_rank()==0) { std::cout << "\n\n\n\n>>>>>>>>>>>>starting SIAL PROGRAM "<< job << std::endl; } //create worker and server sip::DataDistribution data_distribution(sipTables, sip_mpi_attr); sip::GlobalState::set_program_name(prog_name); sip::GlobalState::increment_program(); sip::WorkerPersistentArrayManager wpam; sip::ServerPersistentArrayManager spam; std::cout << "rank " << my_rank << " reached first barrier" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout << "rank " << my_rank << " passed first barrier" << std::endl << std::flush; if (sip_mpi_attr.is_server()) { sip::SIPServer server(sipTables, data_distribution, sip_mpi_attr, &spam); std::cout << "at first barrier in prog 1 at server" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout<<"passed first barrier at server, starting server" << std::endl; server.run(); spam.save_marked_arrays(&server); std::cout << "Server state after termination" << server << std::endl; } else { sip::SialxTimer sialxTimer(sipTables.max_timer_slots()); sip::Interpreter runner(sipTables, sialxTimer, &wpam); std::cout << "at first barrier in prog 1 at worker" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout << "after first barrier; starting worker for "<< job << std::endl; runner.interpret(); wpam.save_marked_arrays(&runner); std::cout << "\n end of prog1 at worker"<< std::endl; } std::cout << std::flush; if (sip_mpi_attr.global_rank()==0) { std::cout << "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" << std::endl << std::flush; std::cout << "SETUP READER DATA FOR SECOND PROGRAM:\n" << setup_reader<< std::endl; } std::string prog_name2 = setup_reader.sial_prog_list_.at(1); setup::BinaryInputFile siox_file2(siox_dir + prog_name2); sip::SipTables sipTables2(setup_reader, siox_file2); if (sip_mpi_attr.global_rank()==0) { std::cout << "SIP TABLES FOR " << prog_name2 << '\n' << sipTables2 << std::endl; } sip::DataDistribution data_distribution2(sipTables2, sip_mpi_attr); sip::GlobalState::set_program_name(prog_name); sip::GlobalState::increment_program(); std::cout << "rank " << my_rank << " reached second barrier in test" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout << "rank " << my_rank << " passed second barrier in test" << std::endl << std::flush; if (sip_mpi_attr.is_server()) { sip::SIPServer server(sipTables2, data_distribution2, sip_mpi_attr, &spam); std::cout << "barrier in prog 2 at server" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout<< "rank " << my_rank << "starting server for prog 2" << std::endl; server.run(); std::cout<< "rank " << my_rank << "Server state after termination of prog2" << server << std::endl; } else { sip::SialxTimer sialxTimer2(sipTables2.max_timer_slots()); sip::Interpreter runner(sipTables2, sialxTimer2, &wpam); std::cout << "rank " << my_rank << "barrier in prog 2 at worker" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout << "rank " << my_rank << "starting worker for prog2"<< job << std::endl; runner.interpret(); std::cout << "\nSIAL PROGRAM 2 TERMINATED"<< std::endl; // Test contents of blocks of distributed array "b" // Get the data for local array block "b" int b_slot = runner.array_slot(std::string("lb")); // Test b(1,1) sip::index_selector_t b_indices_1; b_indices_1[0] = 1; b_indices_1[1] = 1; for (int i = 2; i < MAX_RANK; i++) b_indices_1[i] = sip::unused_index_value; sip::BlockId b_bid_1(b_slot, b_indices_1); std::cout << b_bid_1 << std::endl; sip::Block::BlockPtr b_bptr_1 = runner.get_block_for_reading(b_bid_1); sip::Block::dataPtr b_data_1 = b_bptr_1->get_data(); std::cout << " Comparing block " << b_bid_1 << std::endl; double fill_seq_1_1 = 1.0; for (int i=0; i<segs[0]; i++) { for (int j=0; j<segs[0]; j++) { ASSERT_DOUBLE_EQ(fill_seq_1_1, b_data_1[i*segs[0] + j]); fill_seq_1_1++; } } // Test b(2, 2) sip::index_selector_t b_indices_2; b_indices_2[0] = 2; b_indices_2[1] = 2; for (int i = 2; i < MAX_RANK; i++) b_indices_2[i] = sip::unused_index_value; sip::BlockId b_bid_2(b_slot, b_indices_2); std::cout << b_bid_2 << std::endl; sip::Block::BlockPtr b_bptr_2 = runner.get_block_for_reading(b_bid_2); sip::Block::dataPtr b_data_2 = b_bptr_2->get_data(); std::cout << " Comparing block " << b_bid_2 << std::endl; double fill_seq_2_2 = 4.0; for (int i=0; i<segs[1]; i++) { for (int j=0; j<segs[1]; j++) { ASSERT_DOUBLE_EQ(fill_seq_2_2, b_data_2[i*segs[1] + j]); fill_seq_2_2++; } } // Test b(2,1) sip::index_selector_t b_indices_3; b_indices_3[0] = 2; b_indices_3[1] = 1; for (int i = 2; i < MAX_RANK; i++) b_indices_3[i] = sip::unused_index_value; sip::BlockId b_bid_3(b_slot, b_indices_3); std::cout << b_bid_3 << std::endl; sip::Block::BlockPtr b_bptr_3 = runner.get_block_for_reading(b_bid_3); sip::Block::dataPtr b_data_3 = b_bptr_3->get_data(); std::cout << " Comparing block " << b_bid_3 << std::endl; double fill_seq_2_1 = 3.0; for (int i=0; i<segs[1]; i++) { for (int j=0; j<segs[0]; j++) { ASSERT_DOUBLE_EQ(fill_seq_2_1, b_data_3[i*segs[0] + j]); fill_seq_2_1++; } } } std::cout << "rank " << my_rank << " reached third barrier in test" << std::endl << std::flush; MPI_Barrier(MPI_COMM_WORLD); std::cout << "rank " << my_rank << " passed third barrier in test" << std::endl << std::flush; }
int main(int argc, char* argv[]) { // Default initialization file is data.dat char *init_file = "data.dat"; // Default directory for compiled sialx files is "." char *sialx_file_dir = "."; // Read about getopt here : http://www.gnu.org/software/libc/manual/html_node/Getopt.html // d: means an argument is required for d. Specifies the .dat file. // s: means an argument is required for s. // h & ? are for help. They require no arguments const char *optString = "d:s:h?"; int c; while ((c = getopt(argc, argv, optString)) != -1){ switch (c) { case 'd': init_file = optarg; break; case 's': sialx_file_dir = optarg; break; case 'h':case '?': default: std::cerr<< "Constructs SIP Tables from a .dat file and .siox files and prints them to stdout" <<std::endl; std::cerr<<"Usage : "<<argv[0]<<" -d <init_data_file> -s <sialx_files_directory>"<<std::endl; std::cerr<<"\tDefault data file is \"data.dat\". Default sialx directory is \".\""<<std::endl; std::cerr<<"\t-? or -h to display this usage dialogue"<<std::endl; return 1; } } sip::MemoryTracker::set_global_memory_tracker(new sip::MemoryTracker()); //create setup_file std::string job(init_file); //initialize setup data setup::BinaryInputFile setup_file(job); setup::SetupReader setup_reader(setup_file); setup::SetupReader::SialProgList &progs = setup_reader.sial_prog_list(); setup::SetupReader::SialProgList::iterator it; std::cout << "SetupReader::" <<std::endl << setup_reader << std::endl; for (it = progs.begin(); it != progs.end(); ++it) { std::string sialfpath; sialfpath.append(sialx_file_dir); sialfpath.append("/"); sialfpath.append(*it); setup::BinaryInputFile siox_file(sialfpath); sip::SipTables sipTables(setup_reader, siox_file); // Print the sipTables instance std::cout << "Sial File : " << sialfpath << std::endl; sipTables.print_array_info(std::cout) << std::endl; } std::cout << "Memory used for predefined arrays\n" << *sip::MemoryTracker::global << std::endl <<std::flush; return 0; }
void setup() { setup_reader(); setup_exec(); }