void SialPrinterForTests::do_print_block(const BlockId& id, Block::BlockPtr block, int line_number){ int MAX_TO_PRINT = 1024; int size = block->size(); int OUTPUT_ROW_SIZE = block->shape().segment_sizes_[0]; double* data = block->get_data(); out_.precision(14); out_.setf(std::ios_base::fixed); out_ << line_number << ": "; if (size == 1) { out_ << "printing " << id.str(sip_tables_) << " = "; out_ << *(data); } else { out_ << "printing " << (size < MAX_TO_PRINT?size:MAX_TO_PRINT); out_ << " of " <<size << " elements of block " << id.str(sip_tables_);//BlockId2String(id); out_ << " in the order stored in memory "; int i; for (i = 0; i < size && i < MAX_TO_PRINT; ++i){ if (i%OUTPUT_ROW_SIZE == 0) out_ << std::endl; out_ << *(data+i) << " "; } if (i == MAX_TO_PRINT){ out_ << "...."; } } out_ << std::endl; }
/* gets block, creating it if it doesn't exist. If new, initializes to zero.*/ Block::BlockPtr BlockManager::get_block_for_accumulate(const BlockId& id, bool is_scope_extent) { Block::BlockPtr blk = block(id); if (blk == NULL){ blk = get_block_for_writing(id, is_scope_extent); blk->fill(0.0); } return blk; }
Block::BlockPtr SialOpsParallel::wait_and_check(Block::BlockPtr b, int line) { if (b->state().pending()) { if (sialx_timers_){ sialx_timers_->start_timer(line, SialxTimer::BLOCKWAITTIME); b->state().wait(b->size()); sialx_timers_->pause_timer(line, SialxTimer::BLOCKWAITTIME); } else b->state().wait(b->size()); } return b; }
Block::BlockPtr ContiguousArrayManager::get_block(const BlockId& block_id, int& rank, Block::BlockPtr& contiguous, sip::offset_array_t& offsets) { //get contiguous array that contains block block_id, which must exist, and get its selectors and shape int array_id = block_id.array_id(); rank = sip_tables_.array_rank(array_id); contiguous = get_array(array_id); sip::check(contiguous != NULL, "contiguous array not allocated"); const sip::index_selector_t& selector = sip_tables_.selectors(array_id); BlockShape array_shape = sip_tables_.contiguous_array_shape(array_id); //shape of containing contiguous array //get offsets of block_id in the containing array for (int i = 0; i < rank; ++i) { offsets[i] = sip_tables_.offset_into_contiguous(selector[i], block_id.index_values(i)); } //set offsets of unused indices to 0 std::fill(offsets + rank, offsets + MAX_RANK, 0); //get shape of subblock BlockShape block_shape = sip_tables_.shape(block_id); //allocate a new block and copy data from contiguous block Block::BlockPtr block = new Block(block_shape); contiguous->extract_slice(rank, offsets, block); return block; }
//TODO optimize this. Can reduce searches in block map. void SialOpsParallel::get(BlockId& block_id) { //check for "data race" check_and_set_mode(block_id, READ); //if block already exists, or has pending request, just return Block::BlockPtr block = block_manager_.block(block_id); if (block != NULL) return; //send get message to block's server, and post receive int server_rank = data_distribution_.get_server_rank(block_id); int get_tag; get_tag = barrier_support_.make_mpi_tag_for_GET(); sip::check(server_rank>=0&&server_rank<sip_mpi_attr_.global_size(), "invalid server rank",current_line()); SIP_LOG(std::cout<<"W " << sip_mpi_attr_.global_rank() << " : sending GET for block " << block_id << " to server "<< server_rank << std::endl); // Construct int array to send to server. const int to_send_size = BlockId::MPI_BLOCK_ID_COUNT + 2; const int line_num_offset = BlockId::MPI_BLOCK_ID_COUNT; const int section_num_offset = line_num_offset + 1; int to_send[to_send_size]; // BlockId & line number int *serialized_block_id = block_id.to_mpi_array(); std::copy(serialized_block_id + 0, serialized_block_id + BlockId::MPI_BLOCK_ID_COUNT, to_send); to_send[line_num_offset] = current_line(); to_send[section_num_offset] = barrier_support_.section_number(); SIPMPIUtils::check_err( MPI_Send(to_send, to_send_size, MPI_INT, server_rank, get_tag, MPI_COMM_WORLD)); //allocate block, and insert in block map, using block data as buffer block = block_manager_.get_block_for_writing(block_id, true); //post an asynchronous receive and store the request in the //block's state MPI_Request request; SIPMPIUtils::check_err( MPI_Irecv(block->get_data(), block->size(), MPI_DOUBLE, server_rank, get_tag, MPI_COMM_WORLD, &request)); block->state().mpi_request_ = request; }
/** * A put appears in a SIAL program as * put target(i,j,k,l) += source(i,j,k,l) * So we need the target block id, but the source block data. * Accumulation is done by the server * * The implementation will be more complicated if asynchronous send is * used * * @param target * @param source_ptr */ void SialOpsParallel::put_accumulate(BlockId& target_id, const Block::BlockPtr source_block) { //partial check for data races check_and_set_mode(target_id, WRITE); //send message with target block's id to server int my_rank = sip_mpi_attr_.global_rank(); int server_rank = data_distribution_.get_server_rank(target_id); int put_accumulate_tag, put_accumulate_data_tag; put_accumulate_tag = barrier_support_.make_mpi_tags_for_PUT_ACCUMULATE( put_accumulate_data_tag); sip::check(server_rank>=0&&server_rank<sip_mpi_attr_.global_size(), "invalid server rank",current_line()); SIP_LOG(std::cout<<"W " << sip_mpi_attr_.global_rank() << " : sending PUT_ACCUMULATE for block " << target_id << " to server "<< server_rank << std::endl); // Construct int array to send to server. const int to_send_size = BlockId::MPI_BLOCK_ID_COUNT + 2; const int line_num_offset = BlockId::MPI_BLOCK_ID_COUNT; const int section_num_offset = line_num_offset + 1; int to_send[to_send_size]; // BlockId & line number int *serialized_block_id = target_id.to_mpi_array(); std::copy(serialized_block_id + 0, serialized_block_id + BlockId::MPI_BLOCK_ID_COUNT, to_send); to_send[line_num_offset] = current_line(); to_send[section_num_offset] = barrier_support_.section_number(); //send block id SIPMPIUtils::check_err( MPI_Send(to_send, to_send_size, MPI_INT, server_rank, put_accumulate_tag, MPI_COMM_WORLD)); //immediately follow with the data SIPMPIUtils::check_err( MPI_Send(source_block->get_data(), source_block->size(), MPI_DOUBLE, server_rank, put_accumulate_data_tag, MPI_COMM_WORLD)); //ack ack_handler_.expect_ack_from(server_rank, put_accumulate_data_tag); SIP_LOG( std::cout<< "W " << sip_mpi_attr_.global_rank() << " : Done with PUT_ACCUMULATE for block " << target_id << " to server rank " << server_rank << std::endl); }
void SialPrinterForTests::do_print_contiguous(int array_slot, Block::BlockPtr block, int line_number){ int MAX_TO_PRINT = 1024; int size = block->size(); int OUTPUT_ROW_SIZE = block->shape().segment_sizes_[0]; double* data = block->get_data(); out_ << line_number << ": "; out_ << "printing " << (size < MAX_TO_PRINT?size:MAX_TO_PRINT); out_ << " of " <<size << " elements of contiguous array " << sip_tables_.array_name(array_slot); out_ << " in the order stored in memory "; int i; for (i = 0; i < size && i < MAX_TO_PRINT; ++i){ if (i%OUTPUT_ROW_SIZE == 0) out_ << std::endl; out_.width(14); out_ << *(data+i) << " "; } if (i == MAX_TO_PRINT){ out_ << "...."; } out_ << std::endl; }
Block::BlockPtr ContiguousArrayManager::insert_contiguous_array(int array_id, Block::BlockPtr block_ptr) { sip::check(block_ptr != NULL && block_ptr->get_data() != NULL, "Trying to insert null block_ptr or null block into contiguous array manager\n"); ContiguousArrayMap::iterator it = contiguous_array_map_.find(array_id); if (it != contiguous_array_map_.end()){ delete it->second; it->second = NULL; contiguous_array_map_.erase(it); } contiguous_array_map_[array_id] = block_ptr; SIP_LOG( std::cout<<"Contiguous Block of array "<<sip_tables_.array_name(array_id)<<std::endl); sip::check( block_ptr->shape() == sip_tables_.contiguous_array_shape(array_id), std::string("array ") + sip_tables_.array_name(array_id) + std::string( "shape inconsistent in Sial program and inserted array ")); return block_ptr; }
void BlockManager::lazy_gpu_update_on_host(const Block::BlockPtr& blk) { if (!blk->is_on_gpu() && !blk->is_on_host()) { fail("block allocated neither on host or gpu", current_line()); } else if (!blk->is_on_host()) { blk->allocate_host_data(); _gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size()); } else if (blk->is_dirty_on_gpu()) { _gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size()); } else if (blk->is_dirty_on_all()) { fail("block dirty on host & gpu !", current_line()); } blk->set_on_host(); blk->unset_dirty_on_gpu(); blk->set_dirty_on_host(); }
void BlockManager::lazy_gpu_write_on_host(Block::BlockPtr& blk, const BlockId &id, const BlockShape& shape) { if (!blk->is_on_gpu() && !blk->is_on_host()) { block_map_.cached_delete_block(id); // Get rid of block, create a new one blk = create_block(id, shape); // if (is_scope_extent) { // temp_block_list_stack_.back()->push_back(id); // } } else if (!blk->is_on_host()) { blk->allocate_host_data(); _gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size()); } else if (blk->is_dirty_on_gpu()) { _gpu_device_to_host(blk->get_data(), blk->get_gpu_data(), blk->size()); } else if (blk->is_dirty_on_all()) { fail("block dirty on host & gpu !", current_line()); } blk->set_on_host(); blk->set_dirty_on_host(); blk->unset_dirty_on_gpu(); }
void SialOpsParallel::broadcast_static(Block::BlockPtr source_or_dest, int source_worker){ if (sip_mpi_attr_.num_workers()>0 ){ SIPMPIUtils::check_err(MPI_Bcast(source_or_dest->get_data() , source_or_dest->size(), MPI_DOUBLE, source_worker, sip_mpi_attr_.company_communicator())); } }