TEST(SimpleMPI,persistent_distributed_array_mpi) {
sip::GlobalState::reset_program_count();
sip::SIPMPIAttr &sip_mpi_attr = sip::SIPMPIAttr::get_instance();
int my_rank = sip_mpi_attr.global_rank();
std::cout << "****************************************\n";
sip::DataManager::scope_count=0;
//create setup_file
std::string job("persistent_distributed_array_mpi");
std::cout << "JOBNAME = " << job << std::endl;
double x = 3.456;
int norb = 2;
int segs[] = {2,3};
if (attr.global_rank() == 0) {
init_setup(job.c_str());
set_scalar("x",x);
set_constant("norb",norb);
std::string tmp = job + "1.siox";
const char* nm= tmp.c_str();
add_sial_program(nm);
std::string tmp1 = job + "2.siox";
const char* nm1= tmp1.c_str();
add_sial_program(nm1);
set_aoindex_info(2,segs);
finalize_setup();
}
sip::SIPMPIUtils::check_err(MPI_Barrier(MPI_COMM_WORLD));
setup::BinaryInputFile setup_file(job + ".dat");
setup::SetupReader setup_reader(setup_file);
std::cout << "SETUP READER DATA:\n" << setup_reader<< std::endl;
//get siox name from setup, load and print the sip tables
std::string prog_name = setup_reader.sial_prog_list_.at(0);
std::string siox_dir(dir_name);
setup::BinaryInputFile siox_file(siox_dir + prog_name);
sip::SipTables sipTables(setup_reader, siox_file);
if (!sip_mpi_attr.is_server()) {
std::cout << "SIP TABLES" << '\n' << sipTables << std::endl;
}
if (sip_mpi_attr.global_rank()==0) {
std::cout << "\n\n\n\n>>>>>>>>>>>>starting SIAL PROGRAM "<< job << std::endl;
}
//create worker and server
sip::DataDistribution data_distribution(sipTables, sip_mpi_attr);
sip::GlobalState::set_program_name(prog_name);
sip::GlobalState::increment_program();
sip::WorkerPersistentArrayManager wpam;
sip::ServerPersistentArrayManager spam;
std::cout << "rank " << my_rank << " reached first barrier" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout << "rank " << my_rank << " passed first barrier" << std::endl << std::flush;
if (sip_mpi_attr.is_server()) {
sip::SIPServer server(sipTables, data_distribution, sip_mpi_attr, &spam);
std::cout << "at first barrier in prog 1 at server" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout<<"passed first barrier at server, starting server" << std::endl;
server.run();
spam.save_marked_arrays(&server);
std::cout << "Server state after termination" << server << std::endl;
} else {
sip::SialxTimer sialxTimer(sipTables.max_timer_slots());
sip::Interpreter runner(sipTables, sialxTimer, &wpam);
std::cout << "at first barrier in prog 1 at worker" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout << "after first barrier; starting worker for "<< job << std::endl;
runner.interpret();
wpam.save_marked_arrays(&runner);
std::cout << "\n end of prog1 at worker"<< std::endl;
}
std::cout << std::flush;
if (sip_mpi_attr.global_rank()==0) {
std::cout << "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" << std::endl << std::flush;
std::cout << "SETUP READER DATA FOR SECOND PROGRAM:\n" << setup_reader<< std::endl;
}
std::string prog_name2 = setup_reader.sial_prog_list_.at(1);
setup::BinaryInputFile siox_file2(siox_dir + prog_name2);
sip::SipTables sipTables2(setup_reader, siox_file2);
if (sip_mpi_attr.global_rank()==0) {
std::cout << "SIP TABLES FOR " << prog_name2 << '\n' << sipTables2 << std::endl;
}
sip::DataDistribution data_distribution2(sipTables2, sip_mpi_attr);
sip::GlobalState::set_program_name(prog_name);
sip::GlobalState::increment_program();
std::cout << "rank " << my_rank << " reached second barrier in test" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout << "rank " << my_rank << " passed second barrier in test" << std::endl << std::flush;
if (sip_mpi_attr.is_server()) {
sip::SIPServer server(sipTables2, data_distribution2, sip_mpi_attr, &spam);
std::cout << "barrier in prog 2 at server" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout<< "rank " << my_rank << "starting server for prog 2" << std::endl;
server.run();
std::cout<< "rank " << my_rank << "Server state after termination of prog2" << server << std::endl;
} else {
sip::SialxTimer sialxTimer2(sipTables2.max_timer_slots());
sip::Interpreter runner(sipTables2, sialxTimer2, &wpam);
std::cout << "rank " << my_rank << "barrier in prog 2 at worker" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout << "rank " << my_rank << "starting worker for prog2"<< job << std::endl;
runner.interpret();
std::cout << "\nSIAL PROGRAM 2 TERMINATED"<< std::endl;
// Test contents of blocks of distributed array "b"
// Get the data for local array block "b"
int b_slot = runner.array_slot(std::string("lb"));
// Test b(1,1)
sip::index_selector_t b_indices_1;
b_indices_1[0] = 1;
b_indices_1[1] = 1;
for (int i = 2; i < MAX_RANK; i++) b_indices_1[i] = sip::unused_index_value;
sip::BlockId b_bid_1(b_slot, b_indices_1);
std::cout << b_bid_1 << std::endl;
sip::Block::BlockPtr b_bptr_1 = runner.get_block_for_reading(b_bid_1);
sip::Block::dataPtr b_data_1 = b_bptr_1->get_data();
std::cout << " Comparing block " << b_bid_1 << std::endl;
double fill_seq_1_1 = 1.0;
for (int i=0; i<segs[0]; i++) {
for (int j=0; j<segs[0]; j++) {
ASSERT_DOUBLE_EQ(fill_seq_1_1, b_data_1[i*segs[0] + j]);
fill_seq_1_1++;
}
}
// Test b(2, 2)
sip::index_selector_t b_indices_2;
b_indices_2[0] = 2;
b_indices_2[1] = 2;
for (int i = 2; i < MAX_RANK; i++) b_indices_2[i] = sip::unused_index_value;
sip::BlockId b_bid_2(b_slot, b_indices_2);
std::cout << b_bid_2 << std::endl;
sip::Block::BlockPtr b_bptr_2 = runner.get_block_for_reading(b_bid_2);
sip::Block::dataPtr b_data_2 = b_bptr_2->get_data();
std::cout << " Comparing block " << b_bid_2 << std::endl;
double fill_seq_2_2 = 4.0;
for (int i=0; i<segs[1]; i++) {
for (int j=0; j<segs[1]; j++) {
ASSERT_DOUBLE_EQ(fill_seq_2_2, b_data_2[i*segs[1] + j]);
fill_seq_2_2++;
}
}
// Test b(2,1)
sip::index_selector_t b_indices_3;
b_indices_3[0] = 2;
b_indices_3[1] = 1;
for (int i = 2; i < MAX_RANK; i++) b_indices_3[i] = sip::unused_index_value;
sip::BlockId b_bid_3(b_slot, b_indices_3);
std::cout << b_bid_3 << std::endl;
sip::Block::BlockPtr b_bptr_3 = runner.get_block_for_reading(b_bid_3);
sip::Block::dataPtr b_data_3 = b_bptr_3->get_data();
std::cout << " Comparing block " << b_bid_3 << std::endl;
double fill_seq_2_1 = 3.0;
for (int i=0; i<segs[1]; i++) {
for (int j=0; j<segs[0]; j++) {
ASSERT_DOUBLE_EQ(fill_seq_2_1, b_data_3[i*segs[0] + j]);
fill_seq_2_1++;
}
}
}
std::cout << "rank " << my_rank << " reached third barrier in test" << std::endl << std::flush;
MPI_Barrier(MPI_COMM_WORLD);
std::cout << "rank " << my_rank << " passed third barrier in test" << std::endl << std::flush;
}
int main(int argc, char* argv[]) {
// Default initialization file is data.dat
char *init_file = "data.dat";
// Default directory for compiled sialx files is "."
char *sialx_file_dir = ".";
// Read about getopt here : http://www.gnu.org/software/libc/manual/html_node/Getopt.html
// d: means an argument is required for d. Specifies the .dat file.
// s: means an argument is required for s.
// h & ? are for help. They require no arguments
const char *optString = "d:s:h?";
int c;
while ((c = getopt(argc, argv, optString)) != -1){
switch (c) {
case 'd':
init_file = optarg;
break;
case 's':
sialx_file_dir = optarg;
break;
case 'h':case '?':
default:
std::cerr<< "Constructs SIP Tables from a .dat file and .siox files and prints them to stdout" <<std::endl;
std::cerr<<"Usage : "<<argv[0]<<" -d <init_data_file> -s <sialx_files_directory>"<<std::endl;
std::cerr<<"\tDefault data file is \"data.dat\". Default sialx directory is \".\""<<std::endl;
std::cerr<<"\t-? or -h to display this usage dialogue"<<std::endl;
return 1;
}
}
sip::MemoryTracker::set_global_memory_tracker(new sip::MemoryTracker());
//create setup_file
std::string job(init_file);
//initialize setup data
setup::BinaryInputFile setup_file(job);
setup::SetupReader setup_reader(setup_file);
setup::SetupReader::SialProgList &progs = setup_reader.sial_prog_list();
setup::SetupReader::SialProgList::iterator it;
std::cout << "SetupReader::" <<std::endl << setup_reader << std::endl;
for (it = progs.begin(); it != progs.end(); ++it) {
std::string sialfpath;
sialfpath.append(sialx_file_dir);
sialfpath.append("/");
sialfpath.append(*it);
setup::BinaryInputFile siox_file(sialfpath);
sip::SipTables sipTables(setup_reader, siox_file);
// Print the sipTables instance
std::cout << "Sial File : " << sialfpath << std::endl;
sipTables.print_array_info(std::cout) << std::endl;
}
std::cout << "Memory used for predefined arrays\n" <<
*sip::MemoryTracker::global << std::endl <<std::flush;
return 0;
}
void setup() {
setup_reader();
setup_exec();
}
