ServerBlock* DiskBackedBlockMap::get_block_for_writing(const BlockId& block_id){
/** If block is not in block map, allocate space for it
* Otherwise, if the block is not in memory, allocate space for it.
* Set in_memory and dirty_flag
*/
ServerBlock* block = block_map_.block(block_id);
size_t block_size = sip_tables_.block_size(block_id);
if (block == NULL) {
std::stringstream msg;
msg << "S " << sip_mpi_attr_.global_rank();
msg << " : getting uninitialized block " << block_id << ". Creating zero block for writing"<< std::endl;
SIP_LOG(std::cout << msg.str() << std::flush);
block = allocate_block(NULL, block_size);
block_map_.insert_block(block_id, block);
} else {
if (!block->is_in_memory())
block->allocate_in_memory_data();
}
block->set_in_memory();
block->set_dirty();
policy_.touch(block_id);
return block;
}
ServerBlock* DiskBackedBlockMap::allocate_block(ServerBlock* block, size_t block_size, bool initialize){
/** If enough memory remains, allocates block and returns.
* Otherwise, frees up memory by writing out dirty blocks
* till enough memory has been obtained, then allocates
* and returns block.
*/
std::size_t remaining_mem = max_allocatable_bytes_ - ServerBlock::allocated_bytes();
while (block_size * sizeof(double) > remaining_mem){
try{
BlockId bid = policy_.get_next_block_for_removal();
ServerBlock* blk = block_map_.block(bid);
SIP_LOG(std::cout << "S " << sip_mpi_attr_.company_rank()
<< " : Freeing block " << bid
<< " and writing to disk to make space for new block"
<< std::endl);
if(blk->is_dirty()){
write_block_to_disk(bid, blk);
}
blk->free_in_memory_data();
// Junmin's fix :
// As a result of freeing up block memory, the remaining memory should
// have increased. Otherwise it will go into an infinite loop.
if (!(remaining_mem < max_allocatable_bytes_ - ServerBlock::allocated_bytes())) {
throw std::out_of_range("Break now.");
}
remaining_mem = max_allocatable_bytes_ - ServerBlock::allocated_bytes();
} catch (const std::out_of_range& oor){
std::cerr << " In DiskBackedBlockMap::allocate_block" << std::endl;
std::cerr << oor.what() << std::endl;
std::cerr << *this << std::endl;
fail(" Something got messed up in the internal data structures of the Server", current_line());
} catch(const std::bad_alloc& ba){
std::cerr << " In DiskBackedBlockMap::allocate_block" << std::endl;
std::cerr << ba.what() << std::endl;
std::cerr << *this << std::endl;
fail(" Could not allocate ServerBlock, out of memory", current_line());
}
}
std::stringstream ss;
ss << "S " << sip_mpi_attr_.company_rank() << " : Could not allocate memory for block of size "
<< block_size << ", Memory being used :" << ServerBlock::allocated_bytes() << std::endl;
sip :: check (block_size <= max_allocatable_bytes_ - ServerBlock::allocated_bytes(), ss.str());
if (block == NULL) {
block = new ServerBlock(block_size, initialize);
} else {
block->allocate_in_memory_data();
}
return block;
}
ServerBlock* DiskBackedBlockMap::get_block_for_reading(const BlockId& block_id){
/** If block is not in block map, there is an error !!
* Otherwise, if the block is in memory, read and return or
* if it is only on disk, read it in, store in block map and return.
* Set in_memory flag.
*/
ServerBlock* block = block_map_.block(block_id);
size_t block_size = sip_tables_.block_size(block_id);
if (block == NULL) {
// Error !
std::stringstream errmsg;
errmsg << " S " << sip_mpi_attr_.global_rank();
errmsg << " : Asking for block " << block_id << ". It has not been put/prepared before !"<< std::endl;
std::cout << errmsg.str() << std::flush;
sip::fail(errmsg.str());
// WARNING DISABLED !
if (false){
std::stringstream msg;
msg << "S " << sip_mpi_attr_.global_rank();
msg << " : getting uninitialized block " << block_id << ". Creating zero block "<< std::endl;
std::cout << msg.str() << std::flush;
block = allocate_block(NULL, block_size);
block_map_.insert_block(block_id, block);
}
} else {
if(!block->is_in_memory())
if (block->is_on_disk()){
read_block_from_disk(block, block_id, block_size);
} else {
sip::fail("get_block_for_reading : ServerBlock neither on memory or on disk !");
}
}
block->set_in_memory();
policy_.touch(block_id);
sip::check(block != NULL, "Block is NULL in Server get_block_for_reading, should not happen !");
return block;
}
/** Template specialization for ServerBlock.
*
* Regular LRU Array policy is to evict the
* "first" block from the least recently used array.
* For ServerBlocks however, since they are not
* removed from the BlockMap, but are "emptied out",
* the regular array policy won't work.
*
* This methods first chooses an array whose block
* is to be evicted. It then finds one, skipping
* over those blocks which have been "emptied" out.
*/
template<> BlockId LRUArrayPolicy<ServerBlock>::get_next_block_for_removal(){
/** Return an arbitrary block from the least recently used array
*/
if(lru_list_.empty())
throw std::out_of_range("No blocks have been touched, yet block requested for flushing");
while (!lru_list_.empty()) {
int to_remove_array = lru_list_.back();
IdBlockMap<ServerBlock>::PerArrayMap* array_map = block_map_.per_array_map(to_remove_array);
IdBlockMap<ServerBlock>::PerArrayMap::iterator it = array_map->begin();
for (; it != array_map->end(); it ++) {
ServerBlock *blk = it->second;
if (blk !=NULL && blk->get_data() != NULL) {
return it->first;
}
}
lru_list_.pop_back();
}
throw std::out_of_range("No server blocks to remove - all empty or none present !");
}
//returns the number of doubles actually freed
size_t DiskBackedBlockMap::backup_and_free_doubles(size_t requested_doubles_to_free) {
size_t freed_count = 0;
try {
while (freed_count < requested_doubles_to_free) { //if requested_doubles_to_free <= 0, no iterations performed.
ServerBlock* block;
BlockId bid = policy_.get_next_block_for_removal(block);
block->wait();
ServerBlock* blk = block_map_.block(bid);
check(blk == block, "bug in free_doubles");
SIP_LOG(
std::cout << "S " << sip_mpi_attr_.company_rank() << " : Freeing block " << bid << " and writing to disk to make space for new block" << std::endl);
if (!blk->disk_state_.is_valid_on_disk()) {
write_block_to_disk(bid, blk);
blocks_to_disk_.inc();
blk->disk_state_.set_valid_on_disk();
}
double* data_to_free = blk->get_data();
free_data(data_to_free, blk->size()); //this method updates remaining_doubles_
blk->disk_state_.unset_in_memory();
blk->data_ = NULL;
freed_count += blk->size();
}
} catch (const std::out_of_range& oor) {
//ran out of blocks, just return what was freed.
}
return freed_count;
}
