void SialPrinterForTests::do_print_block(const BlockId& id, Block::BlockPtr block, int line_number){
int MAX_TO_PRINT = 1024;
int size = block->size();
int OUTPUT_ROW_SIZE = block->shape().segment_sizes_[0];
double* data = block->get_data();
out_.precision(14);
out_.setf(std::ios_base::fixed);
out_ << line_number << ": ";
if (size == 1) {
out_ << "printing " << id.str(sip_tables_) << " = ";
out_ << *(data);
} else {
out_ << "printing " << (size < MAX_TO_PRINT?size:MAX_TO_PRINT);
out_ << " of " <<size << " elements of block " << id.str(sip_tables_);//BlockId2String(id);
out_ << " in the order stored in memory ";
int i;
for (i = 0; i < size && i < MAX_TO_PRINT; ++i){
if (i%OUTPUT_ROW_SIZE == 0) out_ << std::endl;
out_ << *(data+i) << " ";
}
if (i == MAX_TO_PRINT){
out_ << "....";
}
}
out_ << std::endl;
}
/* gets block, creating it if it doesn't exist. If new, initializes to zero.*/
Block::BlockPtr BlockManager::get_block_for_accumulate(const BlockId& id, bool is_scope_extent) {
Block::BlockPtr blk = block(id);
if (blk == NULL){
blk = get_block_for_writing(id, is_scope_extent);
blk->fill(0.0);
}
return blk;
}
Block::BlockPtr SialOpsParallel::wait_and_check(Block::BlockPtr b, int line) {
if (b->state().pending()) {
if (sialx_timers_){
sialx_timers_->start_timer(line, SialxTimer::BLOCKWAITTIME);
b->state().wait(b->size());
sialx_timers_->pause_timer(line, SialxTimer::BLOCKWAITTIME);
}
else b->state().wait(b->size());
}
return b;
}
Block::BlockPtr ContiguousArrayManager::get_block(const BlockId& block_id, int& rank,
Block::BlockPtr& contiguous, sip::offset_array_t& offsets) {
//get contiguous array that contains block block_id, which must exist, and get its selectors and shape
int array_id = block_id.array_id();
rank = sip_tables_.array_rank(array_id);
contiguous = get_array(array_id);
sip::check(contiguous != NULL, "contiguous array not allocated");
const sip::index_selector_t& selector = sip_tables_.selectors(array_id);
BlockShape array_shape = sip_tables_.contiguous_array_shape(array_id); //shape of containing contiguous array
//get offsets of block_id in the containing array
for (int i = 0; i < rank; ++i) {
offsets[i] = sip_tables_.offset_into_contiguous(selector[i],
block_id.index_values(i));
}
//set offsets of unused indices to 0
std::fill(offsets + rank, offsets + MAX_RANK, 0);
//get shape of subblock
BlockShape block_shape = sip_tables_.shape(block_id);
//allocate a new block and copy data from contiguous block
Block::BlockPtr block = new Block(block_shape);
contiguous->extract_slice(rank, offsets, block);
return block;
}
//TODO optimize this. Can reduce searches in block map.
void SialOpsParallel::get(BlockId& block_id) {
//check for "data race"
check_and_set_mode(block_id, READ);
//if block already exists, or has pending request, just return
Block::BlockPtr block = block_manager_.block(block_id);
if (block != NULL)
return;
//send get message to block's server, and post receive
int server_rank = data_distribution_.get_server_rank(block_id);
int get_tag;
get_tag = barrier_support_.make_mpi_tag_for_GET();
sip::check(server_rank>=0&&server_rank<sip_mpi_attr_.global_size(), "invalid server rank",current_line());
SIP_LOG(std::cout<<"W " << sip_mpi_attr_.global_rank()
<< " : sending GET for block " << block_id
<< " to server "<< server_rank << std::endl);
// Construct int array to send to server.
const int to_send_size = BlockId::MPI_BLOCK_ID_COUNT + 2;
const int line_num_offset = BlockId::MPI_BLOCK_ID_COUNT;
const int section_num_offset = line_num_offset + 1;
int to_send[to_send_size]; // BlockId & line number
int *serialized_block_id = block_id.to_mpi_array();
std::copy(serialized_block_id + 0, serialized_block_id + BlockId::MPI_BLOCK_ID_COUNT, to_send);
to_send[line_num_offset] = current_line();
to_send[section_num_offset] = barrier_support_.section_number();
SIPMPIUtils::check_err(
MPI_Send(to_send, to_send_size, MPI_INT,
server_rank, get_tag, MPI_COMM_WORLD));
//allocate block, and insert in block map, using block data as buffer
block = block_manager_.get_block_for_writing(block_id, true);
//post an asynchronous receive and store the request in the
//block's state
MPI_Request request;
SIPMPIUtils::check_err(
MPI_Irecv(block->get_data(), block->size(), MPI_DOUBLE, server_rank,
get_tag, MPI_COMM_WORLD, &request));
block->state().mpi_request_ = request;
}
/**
* A put appears in a SIAL program as
* put target(i,j,k,l) += source(i,j,k,l)
* So we need the target block id, but the source block data.
* Accumulation is done by the server
*
* The implementation will be more complicated if asynchronous send is
* used
*
* @param target
* @param source_ptr
*/
void SialOpsParallel::put_accumulate(BlockId& target_id,
const Block::BlockPtr source_block) {
//partial check for data races
check_and_set_mode(target_id, WRITE);
//send message with target block's id to server
int my_rank = sip_mpi_attr_.global_rank();
int server_rank = data_distribution_.get_server_rank(target_id);
int put_accumulate_tag, put_accumulate_data_tag;
put_accumulate_tag = barrier_support_.make_mpi_tags_for_PUT_ACCUMULATE(
put_accumulate_data_tag);
sip::check(server_rank>=0&&server_rank<sip_mpi_attr_.global_size(), "invalid server rank",current_line());
SIP_LOG(std::cout<<"W " << sip_mpi_attr_.global_rank()
<< " : sending PUT_ACCUMULATE for block " << target_id
<< " to server "<< server_rank << std::endl);
// Construct int array to send to server.
const int to_send_size = BlockId::MPI_BLOCK_ID_COUNT + 2;
const int line_num_offset = BlockId::MPI_BLOCK_ID_COUNT;
const int section_num_offset = line_num_offset + 1;
int to_send[to_send_size]; // BlockId & line number
int *serialized_block_id = target_id.to_mpi_array();
std::copy(serialized_block_id + 0, serialized_block_id + BlockId::MPI_BLOCK_ID_COUNT, to_send);
to_send[line_num_offset] = current_line();
to_send[section_num_offset] = barrier_support_.section_number();
//send block id
SIPMPIUtils::check_err(
MPI_Send(to_send, to_send_size, MPI_INT,
server_rank, put_accumulate_tag, MPI_COMM_WORLD));
//immediately follow with the data
SIPMPIUtils::check_err(
MPI_Send(source_block->get_data(), source_block->size(), MPI_DOUBLE,
server_rank, put_accumulate_data_tag, MPI_COMM_WORLD));
//ack
ack_handler_.expect_ack_from(server_rank, put_accumulate_data_tag);
SIP_LOG(
std::cout<< "W " << sip_mpi_attr_.global_rank() << " : Done with PUT_ACCUMULATE for block " << target_id << " to server rank " << server_rank << std::endl);
}
void SialPrinterForTests::do_print_contiguous(int array_slot, Block::BlockPtr block, int line_number){
int MAX_TO_PRINT = 1024;
int size = block->size();
int OUTPUT_ROW_SIZE = block->shape().segment_sizes_[0];
double* data = block->get_data();
out_ << line_number << ": ";
out_ << "printing " << (size < MAX_TO_PRINT?size:MAX_TO_PRINT);
out_ << " of " <<size << " elements of contiguous array " << sip_tables_.array_name(array_slot);
out_ << " in the order stored in memory ";
int i;
for (i = 0; i < size && i < MAX_TO_PRINT; ++i){
if (i%OUTPUT_ROW_SIZE == 0) out_ << std::endl;
out_.width(14);
out_ << *(data+i) << " ";
}
if (i == MAX_TO_PRINT){
out_ << "....";
}
out_ << std::endl;
}
Block::BlockPtr ContiguousArrayManager::insert_contiguous_array(int array_id,
Block::BlockPtr block_ptr) {
sip::check(block_ptr != NULL && block_ptr->get_data() != NULL,
"Trying to insert null block_ptr or null block into contiguous array manager\n");
ContiguousArrayMap::iterator it = contiguous_array_map_.find(array_id);
if (it != contiguous_array_map_.end()){
delete it->second;
it->second = NULL;
contiguous_array_map_.erase(it);
}
contiguous_array_map_[array_id] = block_ptr;
SIP_LOG(
std::cout<<"Contiguous Block of array "<<sip_tables_.array_name(array_id)<<std::endl);
sip::check(
block_ptr->shape() == sip_tables_.contiguous_array_shape(array_id),
std::string("array ") + sip_tables_.array_name(array_id)
+ std::string(
"shape inconsistent in Sial program and inserted array "));
return block_ptr;
}
void BlockManager::lazy_gpu_update_on_host(const Block::BlockPtr& blk) {
if (!blk->is_on_gpu() && !blk->is_on_host()) {
fail("block allocated neither on host or gpu", current_line());
} else if (!blk->is_on_host()) {
blk->allocate_host_data();
_gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size());
} else if (blk->is_dirty_on_gpu()) {
_gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size());
} else if (blk->is_dirty_on_all()) {
fail("block dirty on host & gpu !", current_line());
}
blk->set_on_host();
blk->unset_dirty_on_gpu();
blk->set_dirty_on_host();
}
void BlockManager::lazy_gpu_write_on_host(Block::BlockPtr& blk, const BlockId &id, const BlockShape& shape) {
if (!blk->is_on_gpu() && !blk->is_on_host()) {
block_map_.cached_delete_block(id); // Get rid of block, create a new one
blk = create_block(id, shape);
// if (is_scope_extent) {
// temp_block_list_stack_.back()->push_back(id);
// }
} else if (!blk->is_on_host()) {
blk->allocate_host_data();
_gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size());
} else if (blk->is_dirty_on_gpu()) {
_gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size());
} else if (blk->is_dirty_on_all()) {
fail("block dirty on host & gpu !", current_line());
}
blk->set_on_host();
blk->set_dirty_on_host();
blk->unset_dirty_on_gpu();
}
void SialOpsParallel::broadcast_static(Block::BlockPtr source_or_dest, int source_worker){
if (sip_mpi_attr_.num_workers()>0 ){
SIPMPIUtils::check_err(MPI_Bcast(source_or_dest->get_data() , source_or_dest->size(), MPI_DOUBLE, source_worker,
sip_mpi_attr_.company_communicator()));
}
}
